From 3d62ff8211215127a57e7cfd1842dca225115b6c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 11 Feb 2022 09:48:08 +0000 Subject: [PATCH 001/108] use mmgen launcher; init fixes; blocks-info: improve miner string parsing --- README.md | 2 ++ cmds/mmnode-blocks-info | 17 +++++++++++++++++ cmds/mmnode-feeview | 17 +++++++++++++++++ cmds/mmnode-halving-calculator | 17 +++++++++++++++++ cmds/mmnode-netrate | 17 +++++++++++++++++ cmds/mmnode-peerblocks | 17 +++++++++++++++++ cmds/mmnode-txfind | 17 +++++++++++++++++ mmgen_node_tools/BlocksInfo.py | 13 +++++++++++-- mmgen_node_tools/Sound.py | 0 mmgen_node_tools/Term.py | 0 mmgen_node_tools/Util.py | 0 mmgen_node_tools/__init__.py | 0 mmgen_node_tools/data/version | 2 +- .../main_blocks_info.py | 2 +- .../main_feeview.py | 8 ++++++-- .../main_halving_calculator.py | 2 +- .../main_netrate.py | 0 .../main_peerblocks.py | 0 .../main_txfind.py | 0 setup.cfg | 12 ++++++------ 20 files changed, 130 insertions(+), 13 deletions(-) create mode 100755 cmds/mmnode-blocks-info create mode 100755 cmds/mmnode-feeview create mode 100755 cmds/mmnode-halving-calculator create mode 100755 cmds/mmnode-netrate create mode 100755 cmds/mmnode-peerblocks create mode 100755 cmds/mmnode-txfind mode change 100644 => 100755 mmgen_node_tools/BlocksInfo.py mode change 100644 => 100755 mmgen_node_tools/Sound.py mode change 100644 => 100755 mmgen_node_tools/Term.py mode change 100644 => 100755 mmgen_node_tools/Util.py mode change 100644 => 100755 mmgen_node_tools/__init__.py rename mmnode-blocks-info => mmgen_node_tools/main_blocks_info.py (98%) rename mmnode-feeview => mmgen_node_tools/main_feeview.py (98%) rename mmnode-halving-calculator => mmgen_node_tools/main_halving_calculator.py (99%) rename mmnode-netrate => mmgen_node_tools/main_netrate.py (100%) rename mmnode-peerblocks => mmgen_node_tools/main_peerblocks.py (100%) rename mmnode-txfind => mmgen_node_tools/main_txfind.py (100%) diff --git a/README.md b/README.md index 904c4d4..e8ac441 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ Then, $ python3 -m build --no-isolation $ python3 -m pip install --user dist/*.whl +Also make sure that `~/.local/bin` is in `PATH`. + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - [**Forum**][4] | diff --git a/cmds/mmnode-blocks-info b/cmds/mmnode-blocks-info new file mode 100755 index 0000000..a1c855e --- /dev/null +++ b/cmds/mmnode-blocks-info @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-blocks-info: Display information about a block or range of blocks +""" + +from mmgen.main import launch + +launch('blocks_info',package='mmgen_node_tools') diff --git a/cmds/mmnode-feeview b/cmds/mmnode-feeview new file mode 100755 index 0000000..5a4625e --- /dev/null +++ b/cmds/mmnode-feeview @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-feeview: Visualize the fee structure of a node’s mempool +""" + +from mmgen.main import launch + +launch('feeview',package='mmgen_node_tools') diff --git a/cmds/mmnode-halving-calculator b/cmds/mmnode-halving-calculator new file mode 100755 index 0000000..ebf1f6d --- /dev/null +++ b/cmds/mmnode-halving-calculator @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) +""" + +from mmgen.main import launch + +launch('halving_calculator',package='mmgen_node_tools') diff --git a/cmds/mmnode-netrate b/cmds/mmnode-netrate new file mode 100755 index 0000000..cfefab2 --- /dev/null +++ b/cmds/mmnode-netrate @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-netrate: Bitcoin daemon network rate monitor +""" + +from mmgen.main import launch + +launch('netrate',package='mmgen_node_tools') diff --git a/cmds/mmnode-peerblocks b/cmds/mmnode-peerblocks new file mode 100755 index 0000000..be4bb41 --- /dev/null +++ b/cmds/mmnode-peerblocks @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-peerblocks: List blocks in flight, disconnect stalling nodes +""" + +from mmgen.main import launch + +launch('peerblocks',package='mmgen_node_tools') diff --git a/cmds/mmnode-txfind b/cmds/mmnode-txfind new file mode 100755 index 0000000..05213a8 --- /dev/null +++ b/cmds/mmnode-txfind @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-txfind: Find a transaction in the blockchain or mempool +""" + +from mmgen.main import launch + +launch('txfind',package='mmgen_node_tools') diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py old mode 100644 new mode 100755 index 3dde787..28395b4 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -226,15 +226,18 @@ class BlocksInfo: ) if 'miner' in self.fnames: + # capturing parens must contain only ASCII chars! self.miner_pats = [re.compile(pat) for pat in ( rb'`/([_a-zA-Z0-9&. #/-]+)/', - rb'[\xe3\xe4\xe5][\^/](.*?)\xfa', + rb'[\xe3\xe4\xe5][\^/]([\x20-\x7e]+?)\xfa', rb'([a-zA-Z0-9&. -]+/Mined by [a-zA-Z0-9. ]+)', rb'\x08/(.*Mined by [a-zA-Z0-9. ]+)', rb'Mined by ([a-zA-Z0-9. ]+)', rb'[`]([_a-zA-Z0-9&. #/-]+)[/\xfa]', + rb'([\x20-\x7e]{9,})', rb'[/^]([a-zA-Z0-9&. #/-]{5,})', rb'[/^]([_a-zA-Z0-9&. #/-]+)/', + rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]', )] self.block_data = namedtuple('block_data',self.fnames) @@ -428,10 +431,16 @@ class BlocksInfo: if self.opt.raw_miner_info: return repr(cb) else: + trmap_in = { + '\\': ' ', + '/': ' ', + ',': ' ', + } + trmap = { ord(a):b for a,b in trmap_in.items() } for pat in self.miner_pats: m = pat.search(cb) if m: - return ''.join(chr(b) for b in m[1] if 31 < b < 127).strip('^').strip('/').replace('/',' ') + return re.sub( r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip() ) return '' def print_header(self): diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py old mode 100644 new mode 100755 diff --git a/mmgen_node_tools/Term.py b/mmgen_node_tools/Term.py old mode 100644 new mode 100755 diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py old mode 100644 new mode 100755 diff --git a/mmgen_node_tools/__init__.py b/mmgen_node_tools/__init__.py old mode 100644 new mode 100755 diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4a36342..88a447a 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.0.0 +3.1.dev1 diff --git a/mmnode-blocks-info b/mmgen_node_tools/main_blocks_info.py similarity index 98% rename from mmnode-blocks-info rename to mmgen_node_tools/main_blocks_info.py index fc43706..a7dff8d 100755 --- a/mmnode-blocks-info +++ b/mmgen_node_tools/main_blocks_info.py @@ -21,7 +21,7 @@ mmnode-blocks-info: Display information about a block or range of blocks """ from mmgen.common import * -from mmgen_node_tools.BlocksInfo import BlocksInfo,JSONBlocksInfo +from .BlocksInfo import BlocksInfo,JSONBlocksInfo opts_data = { 'sets': [ diff --git a/mmnode-feeview b/mmgen_node_tools/main_feeview.py similarity index 98% rename from mmnode-feeview rename to mmgen_node_tools/main_feeview.py index c91fc34..8979e44 100755 --- a/mmnode-feeview +++ b/mmgen_node_tools/main_feeview.py @@ -125,9 +125,12 @@ def create_data(coin_amt,mempool): break # remove empty top brackets: - while out[-1].tx_bytes == 0: + while out and out[-1].tx_bytes == 0: out.pop() + if not out: + die(1,'No data!') + out.reverse() # cumulative totals and display are top-down # calculate cumulative byte totals, filter rows: @@ -197,7 +200,7 @@ def gen_body(data): async def main(): from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts() + proto = init_proto_from_opts(need_amt=True) from mmgen.rpc import rpc_init c = await rpc_init(proto) @@ -210,6 +213,7 @@ async def main(): log(mempool,'mempool.json') data = create_data(proto.coin_amt,mempool) + (do_pager if opt.pager else print)( '\n'.join(gen_header(c.host,await c.call('getblockcount'))) + '\n' + '\n'.join(gen_body(data)) diff --git a/mmnode-halving-calculator b/mmgen_node_tools/main_halving_calculator.py similarity index 99% rename from mmnode-halving-calculator rename to mmgen_node_tools/main_halving_calculator.py index 7ddf0a1..423df0e 100755 --- a/mmnode-halving-calculator +++ b/mmgen_node_tools/main_halving_calculator.py @@ -62,7 +62,7 @@ def time_diff_warning(t_diff): async def main(): from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts() + proto = init_proto_from_opts(need_amt=True) from mmgen.rpc import rpc_init c = await rpc_init(proto) diff --git a/mmnode-netrate b/mmgen_node_tools/main_netrate.py similarity index 100% rename from mmnode-netrate rename to mmgen_node_tools/main_netrate.py diff --git a/mmnode-peerblocks b/mmgen_node_tools/main_peerblocks.py similarity index 100% rename from mmnode-peerblocks rename to mmgen_node_tools/main_peerblocks.py diff --git a/mmnode-txfind b/mmgen_node_tools/main_txfind.py similarity index 100% rename from mmnode-txfind rename to mmgen_node_tools/main_txfind.py diff --git a/setup.cfg b/setup.cfg index eeafdef..49c64c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,14 +23,14 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.0.0 + mmgen>=13.1.dev19 packages = mmgen_node_tools scripts = - mmnode-blocks-info - mmnode-feeview - mmnode-halving-calculator - mmnode-netrate - mmnode-peerblocks + cmds/mmnode-blocks-info + cmds/mmnode-feeview + cmds/mmnode-halving-calculator + cmds/mmnode-netrate + cmds/mmnode-peerblocks From 251660e3fe196d7376005a4c13fb142c4b61df02 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 31 May 2022 10:35:34 +0000 Subject: [PATCH 002/108] mmnode-feeview: support Bitcoin Core 23.0 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 88a447a..544b443 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev1 +3.1.dev2 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 8979e44..276c870 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -117,7 +117,7 @@ def create_data(coin_amt,mempool): # populate fee brackets: for tx in mempool.values(): - fee = coin_amt(tx['fee']).to_unit('satoshi') + fee = coin_amt(tx['fees']['base']).to_unit('satoshi') vsize = tx['vsize'] for bracket in out: if fee / vsize < bracket.top: From a01c9856b1976616db9ff29898abca7b36461e75 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 27 Jul 2022 16:16:41 +0000 Subject: [PATCH 003/108] support testing via MMGen test suite framework --- README.md | 11 ++++- setup.cfg | 3 +- test/init.sh | 92 ++++++++++++++++++++++++++++++++++++ test/test-release.d/cfg.sh | 62 ++++++++++++++++++++++++ test/test-release.sh | 79 ------------------------------- test/test_py_d/cfg.py | 28 +++++++++++ test/test_py_d/ts_misc.py | 50 ++++++++++++++++++++ test/test_py_d/ts_regtest.py | 64 +++++++++++++++++++++++++ 8 files changed, 308 insertions(+), 81 deletions(-) create mode 100755 test/init.sh create mode 100755 test/test-release.d/cfg.sh delete mode 100755 test/test-release.sh create mode 100755 test/test_py_d/cfg.py create mode 100755 test/test_py_d/ts_misc.py create mode 100755 test/test_py_d/ts_regtest.py diff --git a/README.md b/README.md index e8ac441..088a344 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# MMGen node tools +# MMGen Node Tools Helper utilities for Bitcoin and forkcoin full nodes. @@ -19,6 +19,15 @@ Then, Also make sure that `~/.local/bin` is in `PATH`. +## Test: + +*NOTE: the tests require that the MMGen and MMGen Node Tools repositories be +located in the same directory.* + + $ test/init.sh + $ test/test-release.sh -A # BTC-only testing + $ test/test-release.sh # Full testing + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - [**Forum**][4] | diff --git a/setup.cfg b/setup.cfg index 49c64c6..638546a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,10 +23,11 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.1.dev19 + mmgen>=13.2.dev10 packages = mmgen_node_tools + mmgen_node_tools.data scripts = cmds/mmnode-blocks-info diff --git a/test/init.sh b/test/init.sh new file mode 100755 index 0000000..0979c76 --- /dev/null +++ b/test/init.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +RED="\e[31;1m" GREEN="\e[32;1m" YELLOW="\e[33;1m" BLUE="\e[34;1m" RESET="\e[0m" + +set -o errtrace +set -o functrace + +trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT +trap 'echo -e "${RED}Node Tools test suite initialization exited with error (line $BASH_LINENO) $RESET"' ERR +umask 0022 + +PROGNAME=$(basename $0) +while getopts h OPT +do + case "$OPT" in + h) printf " %-16s Initialize the MMGen Node Tools test suite\n" "${PROGNAME}:" + echo " USAGE: $PROGNAME" + echo " OPTIONS: '-h' Print this help message" + exit ;; + *) exit ;; + esac +done + +shift $((OPTIND-1)) + +mm_repo='../mmgen' + +die() { echo -e ${YELLOW}ERROR: $1$RESET; false; } +becho() { echo -e $BLUE$1$RESET; } + +check_mmgen_repo() { + ( cd $mm_repo; python3 ./setup.py --url | grep -iq 'mmgen' ) +} + +build_mmgen_extmod() { + ( cd $mm_repo; python3 ./setup.py build_ext --inplace ) +} + +create_dir_links() { + for target in 'mmgen' 'scripts'; do + src="$mm_repo/$target" + if [ -e $target ]; then + [ $(realpath --relative-to=. $target) == $src ] || die "'$target' does not point to '$src'" + else + echo "Creating symlink: $target" + ln -s $src + fi + done +} + +create_test_links() { + sources=' + test/include + test/overlay + test/__init__.py + test/test.py + test/unit_tests.py + test/test-release.sh + test/test_py_d/common.py + test/test_py_d/ts_base.py + cmds/mmgen-regtest + ' + for src in $sources; do + pfx=$(echo $src | sed -r 's/[^/]//g' | sed 's/\//..\//g') + if [ ! -e $src ]; then + echo "Creating symlink: $src" + ( cd "$(dirname $src)" && ln -s "$pfx$mm_repo/$src" ) + fi + done +} + +set -e + +becho 'Initializing MMGen Node Tools Test Suite' + +check_mmgen_repo || die "MMGen repository not found at $mm_repo!" + +build_mmgen_extmod + +create_dir_links + +create_test_links + +becho 'OK' diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh new file mode 100755 index 0000000..9044673 --- /dev/null +++ b/test/test-release.d/cfg.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +list_avail_tests() { + echo "AVAILABLE TESTS:" + echo " unit - unit tests" + echo " btc_rt - Bitcoin regtest" + echo " bch_rt - Bitcoin Cash Node (BCH) regtest" + echo " ltc_rt - Litecoin regtest" + echo " misc - miscellaneous tests that don't fit in the above categories" + echo + echo "AVAILABLE TEST GROUPS:" + echo " default - All tests minus the extra tests" + echo " extra - All tests minus the default tests" + echo " noalt - BTC-only tests" + echo " quick - Default tests minus btc_tn, bch, bch_rt, ltc and ltc_rt" + echo " qskip - The tests skipped in the 'quick' test group" + echo + echo "By default, all tests are run" +} + +init_groups() { + dfl_tests='unit misc btc_rt bch_rt ltc_rt' + extra_tests='' + noalt_tests='unit misc btc_rt' + quick_tests='unit misc btc_rt' + qskip_tests='bch_rt ltc_rt' +} + +init_tests() { + i_unit='Unit' + s_unit="The following tests will test various low-level subsystems" + t_unit="- $unit_tests_py --node-tools" + f_unit='Unit tests completed' + + i_misc='Misc' + s_misc="The following tests will test miscellaneous script features" + t_misc="- $test_py helpscreens" + f_misc='Misc tests completed' + + i_btc_rt='Bitcoin regtest' + s_btc_rt="The following tests will test MMGen's regtest (Bob and Alice) mode" + t_btc_rt="- $test_py regtest" + f_btc_rt='Regtest (Bob and Alice) mode tests for BTC completed' + + i_bch_rt='BitcoinCashNode (BCH) regtest' + s_bch_rt="The following tests will test MMGen's regtest (Bob and Alice) mode" + t_bch_rt="- $test_py --coin=bch regtest" + f_bch_rt='Regtest (Bob and Alice) mode tests for BCH completed' + + i_ltc_rt='Litecoin regtest' + s_ltc_rt="The following tests will test MMGen's regtest (Bob and Alice) mode" + t_ltc_rt="- $test_py --coin=ltc regtest" + f_ltc_rt='Regtest (Bob and Alice) mode tests for LTC completed' +} diff --git a/test/test-release.sh b/test/test-release.sh deleted file mode 100755 index 1a4531a..0000000 --- a/test/test-release.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -export MMGEN_TEST_SUITE=1 -export PYTHONPATH=. - -orig_pwd=$(pwd) - -RED="\e[31;1m" GREEN="\e[32;1m" YELLOW="\e[33;1m" BLUE="\e[34;1m" MAGENTA="\e[35;1m" CYAN="\e[36;1m" -RESET="\e[0m" - -set -o errtrace -set -o functrace - -trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT -trap 'echo -e "${RED}Node tools test suite exited with error$RESET"' ERR -umask 0022 - -unit_tests_py='test/unit_tests.py --names --quiet' - -PROGNAME=$(basename $0) -while getopts hv OPT -do - case "$OPT" in - h) printf " %-16s The MMGen node tools test suite\n" "${PROGNAME}:" - echo " USAGE: $PROGNAME [options] [tests or test group]" - echo " OPTIONS: '-h' Print this help message" - echo " '-v' Run commands with '--verbose' switch" - exit ;; - v) VERBOSE=1 - unit_tests_py="${unit_tests_py/--quiet/--verbose}" ;; - *) exit ;; - esac -done - -shift $((OPTIND-1)) - -nt_repo='../mmgen-node-tools' -mm_repo='../mmgen' - -die() { echo -e $YELLOW$1$RESET; false; } -gecho() { echo -e $GREEN$1$RESET; } -pecho() { echo -e $MAGENTA$1$RESET; } -becho() { echo -e $BLUE$1$RESET; } - -check_mmgen_repo() { - ( cd $mm_repo; python3 ./setup.py --url | grep -iq 'mmgen' ) -} - -create_links() { - ( cd $mm_repo && [ -L 'mmgen_node_tools' ] || ln -s "$orig_pwd/mmgen_node_tools" ) - ( - cd 'test/unit_tests_d' - for fn in ../../$nt_repo/test/unit_tests_d/nt_*.py; do - [ -L "$(basename $fn)" ] || ln -s "$fn" - done - ) -} - -run_unit_tests() { - pecho 'Running unit tests:' - $unit_tests_py --node-tools - pecho 'Completed unit tests' -} - -# start execution - -set -e - -becho 'Starting node tools test suite (WIP)' - -check_mmgen_repo || die "No MMGen repository found at $mm_repo!" - -cd $mm_repo - -create_links - -run_unit_tests - -becho 'Node tools test suite completed successfully' diff --git a/test/test_py_d/cfg.py b/test/test_py_d/cfg.py new file mode 100755 index 0000000..dbb792f --- /dev/null +++ b/test/test_py_d/cfg.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +test.test_py_d.cfg: configuration data for test.py +""" + +import os + +cmd_groups_dfl = { + 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), + 'regtest': ('TestSuiteRegtest',{}), +} + +cmd_groups_extra = {} + +cfgs = { + '1': {}, # regtest +} + +def fixup_cfgs(): pass diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py new file mode 100755 index 0000000..07d52cf --- /dev/null +++ b/test/test_py_d/ts_misc.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +ts_misc.py: Miscellaneous test groups for the test.py test suite +""" + +from mmgen.globalvars import g +from ..include.common import * +from .common import * +from .ts_base import * + +class TestSuiteHelp(TestSuiteBase): + 'help, info and usage screens' + networks = ('btc','ltc','bch') + tmpdir_nums = [] + passthru_opts = ('daemon_data_dir','rpc_port','coin','testnet') + cmd_group = ( + ('version', (1,'version message',[])), + ('helpscreens', (1,'help screens', [])), + ('longhelpscreens', (1,'help screens (--longhelp)',[])), + ) + + def version(self): + t = self.spawn(f'mmnode-netrate',['--version']) + t.expect('MMNODE-NETRATE version') + return t + + def helpscreens(self,arg='--help',scripts=(),expect='USAGE:.*OPTIONS:'): + + scripts = list(scripts) or [s for s in os.listdir('cmds') if s.startswith('mmnode-')] + + for s in sorted(scripts): + t = self.spawn(s,[arg],extra_desc=f'({s})') + t.expect(expect,regex=True) + t.read() + t.ok() + t.skip_ok = True + + return t + + def longhelpscreens(self): + return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py new file mode 100755 index 0000000..5f3ffaf --- /dev/null +++ b/test/test_py_d/ts_regtest.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +ts_regtest.py: Regtest tests for the test.py test suite +""" + +import os +from mmgen.opts import opt +from mmgen.util import die,gmsg +from ..include.common import * +from .common import * + +from .ts_base import * + +class TestSuiteRegtest(TestSuiteBase): + 'various operations via regtest mode' + networks = ('btc','ltc','bch') + passthru_opts = ('coin',) + extra_spawn_args = ['--regtest=1'] + tmpdir_nums = [1] + color = True + deterministic = False + cmd_group = ( + ('setup', 'regtest (Bob and Alice) mode setup'), + ('stop', 'stopping regtest daemon'), + ) + + def __init__(self,trunner,cfgs,spawn): + TestSuiteBase.__init__(self,trunner,cfgs,spawn) + if trunner == None: + return + if self.proto.testnet: + die(2,'--testnet and --regtest options incompatible with regtest test suite') + + os.environ['MMGEN_TEST_SUITE_REGTEST'] = '1' + + def __del__(self): + os.environ['MMGEN_TEST_SUITE_REGTEST'] = '' + + def setup(self): + stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) + from shutil import rmtree + try: rmtree(joinpath(self.tr.data_dir,'regtest')) + except: pass + t = self.spawn('mmgen-regtest',['-n','setup']) + for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): + t.expect(s) + return t + + def stop(self): + if opt.no_daemon_stop: + self.spawn('',msg_only=True) + msg_r('(leaving daemon running by user request)') + return 'ok' + else: + return self.spawn('mmgen-regtest',['stop']) From 6c370dec4bae8f067b969f61e68a14087bab01a7 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 27 Jul 2022 16:16:42 +0000 Subject: [PATCH 004/108] new command: `mmnode-addrbal` Get balances for arbitrary addresses in the blockchain Testing: $ test/test.py -e regtest Example: # Top 10 Bitcoin addresses by balance (see https://blockchair.com/bitcoin/addresses) $ top10=' 34xp4vRoCGJym3xR7yCVPFHoCNxv4Twseo bc1qgdjqv0av3q56jvd82tkdjpy7gdp9ut8tlqmgrpmv24sq90ecnvqqjwvw97 1LQoWist8KkaUXSPKZHNvEyfrEkPHzSsCd 3LYJfcfHPXYJreMsASk2jkn69LWEYKzexb 3M219KR5vEneNb47ewrPfWyb5jQ2DjxRP6 bc1qazcm763858nkj2dj986etajv6wquslv8uxwczt 37XuVSEpWW4trkfmvWzegTHQt7BdktSKUs 1FeexV6bAHb8ybZjqQMjJrcCrHGW9sb6uF bc1qa5wkgaew2dkv56kfvj49j0av5nml45x9ek9hz6 3Kzh9qAqVWQhEsfQz7zEQL1EuSx5tyNLNS' # Compact summary: $ mmnode-addrbal --tabular $top10 # Full output: $ mmnode-addrbal $top10 --- cmds/mmnode-addrbal | 17 ++++ mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 170 +++++++++++++++++++++++++++++++ setup.cfg | 1 + test/test_py_d/ts_regtest.py | 60 +++++++++++ 5 files changed, 249 insertions(+), 1 deletion(-) create mode 100755 cmds/mmnode-addrbal create mode 100755 mmgen_node_tools/main_addrbal.py diff --git a/cmds/mmnode-addrbal b/cmds/mmnode-addrbal new file mode 100755 index 0000000..3a5bc68 --- /dev/null +++ b/cmds/mmnode-addrbal @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-addrbal: Get balances for arbitrary addresses in the blockchain +""" + +from mmgen.main import launch + +launch('addrbal',package='mmgen_node_tools') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 544b443..59afffe 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev2 +3.1.dev3 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py new file mode 100755 index 0000000..c638ed6 --- /dev/null +++ b/mmgen_node_tools/main_addrbal.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-addrbal: Get balances for arbitrary addresses in the blockchain +""" + +from mmgen.common import * + +opts_data = { + 'text': { + 'desc': 'Get balances for arbitrary addresses in the blockchain', + 'usage': '[opts] address [address..]', + 'options': """ +-h, --help Print this help message +--, --longhelp Print help message for long options (common options) +-f, --first-block With tabular output, additionally display first block info +-t, --tabular Produce compact tabular output +""" + } +} + +def do_output(proto,addr_data,blk_hdrs): + + col1w = len(str(len(addr_data))) + indent = ' ' * (col1w + 2) + + for n,(addr,unspents) in enumerate(addr_data.items(),1): + Msg(f'\n{n:{col1w}}) Address: {addr.hl()}') + + if unspents: + heights = { u['height'] for u in unspents } + Msg('{}Balance: {}'.format( + indent, + proto.coin_amt(sum(u['amount'] for u in unspents)).hl2(unit=True,fs='{:,}') )), + Msg('{}{} unspent output{} in {} block{}'.format( + indent, + red(str(len(unspents))), + suf(unspents), + red(str(len(heights))), + suf(heights) )) + blk_w = len(str(unspents[-1]['height'])) + fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent,max(5,blk_w)) + Msg(fs.format('Block','Date','TxID','Vout',' Amount')) + for u in unspents: + Msg(fs.format( + u['height'], + make_timestr( blk_hdrs[u['height']]['time'] ), + CoinTxID(u['txid']).hl(), + red(str(u['vout']).rjust(4)), + proto.coin_amt(u['amount']).fmt(color=True,fs='6.8') + )) + else: + Msg(f'{indent}No balance') + +def do_output_tabular(proto,addr_data,blk_hdrs): + + col1w = len(str(len(addr_data))) + 1 + max_addrw = max(len(addr) for addr in addr_data) + fb_heights = [str(unspents[0]['height']) if unspents else '' for unspents in addr_data.values()] + lb_heights = [str(unspents[-1]['height']) if unspents else '' for unspents in addr_data.values()] + fb_w = max(len(h) for h in fb_heights) + lb_w = max(len(h) for h in lb_heights) + + fs = ( + ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w,max(5,fb_w),max(4,lb_w)) + if opt.first_block else + ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w,max(4,lb_w)) ) + + Msg('\n' + fs.format( + n = '', + a = 'Address'.ljust(max_addrw), + u = 'UTXOs', + b = 'First', + t = 'Block', + B = 'Last', + T = 'Block', + A = ' Amount' )) + + for n,(addr,unspents) in enumerate(addr_data.items(),1): + if unspents: + Msg(fs.format( + n = str(n) + ')', + a = addr.fmt(width=max_addrw,color=True), + u = red(str(len(unspents)).rjust(5)), + b = unspents[0]['height'], + t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), + B = unspents[-1]['height'], + T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), + A = proto.coin_amt(sum(u['amount'] for u in unspents)).fmt(color=True,fs='7.8') + )) + else: + Msg(fs.format( + n = str(n) + ')', + a = addr.fmt(width=max_addrw,color=True), + u = ' -', + b = '-', + t = '', + B = '-', + T = '', + A = ' -' )) + +async def main(req_addrs): + + from mmgen.protocol import init_proto_from_opts + proto = init_proto_from_opts(need_amt=True) + + from mmgen.addr import CoinAddr + addrs = [CoinAddr(proto,addr) for addr in req_addrs] + + from mmgen.rpc import rpc_init + rpc = await rpc_init(proto) + + height = await rpc.call('getblockcount') + Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') + + from mmgen.base_proto.bitcoin.misc import scantxoutset + res = await scantxoutset( rpc, [f'addr({addr})' for addr in addrs] ) + + if not res['success']: + die(1,'UTXO scanning failed or was interrupted') + elif not res['unspents']: + msg('Address has no balance' if len(addrs) == 1 else + 'Addresses have no balances' ) + else: + addr_data = {k:[] for k in addrs} + + if 'desc' in res['unspents'][0]: + import re + for unspent in sorted(res['unspents'],key=lambda x: x['height']): + addr = re.match('addr\((.*?)\)',unspent['desc'])[1] + addr_data[addr].append(unspent) + else: + from mmgen.base_proto.bitcoin.tx.base import scriptPubKey2addr + for unspent in sorted(res['unspents'],key=lambda x: x['height']): + addr = scriptPubKey2addr( proto, unspent['scriptPubKey'] )[0] + addr_data[addr].append(unspent) + + good_addrs = len([v for v in addr_data.values() if v]) + + Msg('Total: {} in {} address{}'.format( + proto.coin_amt(res['total_amount']).hl2(unit=True,fs='{:,}'), + red(str(good_addrs)), + suf(good_addrs,'es') + )) + + blk_heights = {i['height'] for i in res['unspents']} + blk_hashes = await rpc.batch_call('getblockhash', [(h,) for h in blk_heights]) + blk_hdrs = await rpc.batch_call('getblockheader', [(H,) for H in blk_hashes]) + + (do_output_tabular if opt.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) + +cmd_args = opts.init(opts_data,init_opts={'rpc_backend':'aiohttp'}) + +if len(cmd_args) < 1: + die(1,'This command requires at least one coin address argument') + +from mmgen.obj import CoinTxID,Int + +try: + run_session(main(cmd_args)) +except KeyboardInterrupt: + sys.stderr.write('\n') diff --git a/setup.cfg b/setup.cfg index 638546a..3f804e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ packages = mmgen_node_tools.data scripts = + cmds/mmnode-addrbal cmds/mmnode-blocks-info cmds/mmnode-feeview cmds/mmnode-halving-calculator diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 5f3ffaf..0cbc718 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -13,13 +13,25 @@ ts_regtest.py: Regtest tests for the test.py test suite """ import os +from mmgen.globalvars import g from mmgen.opts import opt from mmgen.util import die,gmsg +from mmgen.protocol import init_proto from ..include.common import * from .common import * from .ts_base import * +args1 = ['--bob'] +args2 = ['--bob','--rpc-backend=http'] + +def gen_addrs(proto,network,keys): + from mmgen.tool.api import tool_api + tool = tool_api() + tool.init_coin(proto.coin,'regtest') + tool.addrtype = proto.mmtypes[-1] + return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] + class TestSuiteRegtest(TestSuiteBase): 'various operations via regtest mode' networks = ('btc','ltc','bch') @@ -30,6 +42,18 @@ class TestSuiteRegtest(TestSuiteBase): deterministic = False cmd_group = ( ('setup', 'regtest (Bob and Alice) mode setup'), + ('sendto1', 'sending funds to address #1 (1)'), + ('sendto2', 'sending funds to address #1 (2)'), + ('sendto3', 'sending funds to address #2'), + ('addrbal_single', 'getting address balance (single address)'), + ('addrbal_multiple', 'getting address balances (multiple addresses)'), + ('addrbal_multiple_tabular1', 'getting address balances (multiple addresses, tabular output)'), + ('addrbal_multiple_tabular2', 'getting address balances (multiple addresses, tabular, show first block)'), + ('addrbal_nobal1', 'getting address balances (no balance)'), + ('addrbal_nobal2', 'getting address balances (no balances)'), + ('addrbal_nobal3', 'getting address balances (one null balance)'), + ('addrbal_nobal3_tabular1', 'getting address balances (one null balance, tabular output)'), + ('addrbal_nobal3_tabular2', 'getting address balances (one null balance, tabular, show first block)'), ('stop', 'stopping regtest daemon'), ) @@ -39,6 +63,8 @@ class TestSuiteRegtest(TestSuiteBase): return if self.proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') + self.proto = init_proto(self.proto.coin,network='regtest',need_amt=True) + self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) os.environ['MMGEN_TEST_SUITE_REGTEST'] = '1' @@ -55,6 +81,40 @@ class TestSuiteRegtest(TestSuiteBase): t.expect(s) return t + def sendto(self,addr,amt): + return self.spawn('mmgen-regtest',['send',addr,amt]) + + def sendto1(self): return self.sendto(self.addrs[0],'0.123') + def sendto2(self): return self.sendto(self.addrs[0],'0.234') + def sendto3(self): return self.sendto(self.addrs[1],'0.345') + + def addrbal_single(self): + return self.spawn('mmnode-addrbal',args2+[self.addrs[0]]) + + def addrbal_multiple(self): + return self.spawn('mmnode-addrbal',args2+[self.addrs[1],self.addrs[0]]) + + def addrbal_multiple_tabular1(self): + return self.spawn('mmnode-addrbal',args2+['--tabular',self.addrs[1],self.addrs[0]]) + + def addrbal_multiple_tabular2(self): + return self.spawn('mmnode-addrbal',args1+['--tabular','--first-block',self.addrs[1],self.addrs[0]]) + + def addrbal_nobal1(self): + return self.spawn('mmnode-addrbal',args2+[self.addrs[2]]) + + def addrbal_nobal2(self): + return self.spawn('mmnode-addrbal',args2+[self.addrs[2],self.addrs[3]]) + + def addrbal_nobal3(self): + return self.spawn('mmnode-addrbal',args2+[self.addrs[4],self.addrs[0],self.addrs[3]]) + + def addrbal_nobal3_tabular1(self): + return self.spawn('mmnode-addrbal',args2+['--tabular',self.addrs[4],self.addrs[0],self.addrs[3]]) + + def addrbal_nobal3_tabular2(self): + return self.spawn('mmnode-addrbal',args1+['--tabular','--first-block',self.addrs[4],self.addrs[0],self.addrs[3]]) + def stop(self): if opt.no_daemon_stop: self.spawn('',msg_only=True) From 440bf142418a906bb2cd1f637fdccb84370b6219 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 4 Aug 2022 14:16:26 +0000 Subject: [PATCH 005/108] test suite: minor fixes and cleanups --- test/test-release.d/cfg.sh | 14 +++++++------- test/test_py_d/cfg.py | 4 ++-- test/test_py_d/ts_misc.py | 2 +- .../{nt_BlocksInfo.py => ut_BlocksInfo.py} | 0 4 files changed, 10 insertions(+), 10 deletions(-) rename test/unit_tests_d/{nt_BlocksInfo.py => ut_BlocksInfo.py} (100%) diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 9044673..fdc4800 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -37,7 +37,7 @@ init_groups() { init_tests() { i_unit='Unit' s_unit="The following tests will test various low-level subsystems" - t_unit="- $unit_tests_py --node-tools" + t_unit="- $unit_tests_py" f_unit='Unit tests completed' i_misc='Misc' @@ -46,17 +46,17 @@ init_tests() { f_misc='Misc tests completed' i_btc_rt='Bitcoin regtest' - s_btc_rt="The following tests will test MMGen's regtest (Bob and Alice) mode" + s_btc_rt="The following tests will test various scripts using regtest mode" t_btc_rt="- $test_py regtest" - f_btc_rt='Regtest (Bob and Alice) mode tests for BTC completed' + f_btc_rt='Regtest mode tests for BTC completed' i_bch_rt='BitcoinCashNode (BCH) regtest' - s_bch_rt="The following tests will test MMGen's regtest (Bob and Alice) mode" + s_bch_rt="The following tests will test various scripts using regtest mode" t_bch_rt="- $test_py --coin=bch regtest" - f_bch_rt='Regtest (Bob and Alice) mode tests for BCH completed' + f_bch_rt='Regtest mode tests for BCH completed' i_ltc_rt='Litecoin regtest' - s_ltc_rt="The following tests will test MMGen's regtest (Bob and Alice) mode" + s_ltc_rt="The following tests will test various scripts using regtest mode" t_ltc_rt="- $test_py --coin=ltc regtest" - f_ltc_rt='Regtest (Bob and Alice) mode tests for LTC completed' + f_ltc_rt='Regtest mode tests for LTC completed' } diff --git a/test/test_py_d/cfg.py b/test/test_py_d/cfg.py index dbb792f..c56f74c 100755 --- a/test/test_py_d/cfg.py +++ b/test/test_py_d/cfg.py @@ -15,8 +15,8 @@ test.test_py_d.cfg: configuration data for test.py import os cmd_groups_dfl = { - 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), - 'regtest': ('TestSuiteRegtest',{}), + 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), + 'regtest': ('TestSuiteRegtest',{}), } cmd_groups_extra = {} diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 07d52cf..707f32f 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -12,7 +12,6 @@ ts_misc.py: Miscellaneous test groups for the test.py test suite """ -from mmgen.globalvars import g from ..include.common import * from .common import * from .ts_base import * @@ -27,6 +26,7 @@ class TestSuiteHelp(TestSuiteBase): ('helpscreens', (1,'help screens', [])), ('longhelpscreens', (1,'help screens (--longhelp)',[])), ) + color = True def version(self): t = self.spawn(f'mmnode-netrate',['--version']) diff --git a/test/unit_tests_d/nt_BlocksInfo.py b/test/unit_tests_d/ut_BlocksInfo.py similarity index 100% rename from test/unit_tests_d/nt_BlocksInfo.py rename to test/unit_tests_d/ut_BlocksInfo.py From ba28bffc5c538870a020773049eee8ce70929411 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 4 Aug 2022 14:16:27 +0000 Subject: [PATCH 006/108] `test.py`: addr `addrbal` subgroup, improve tests --- test/test_py_d/ts_regtest.py | 89 ++++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 0cbc718..a6836be 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -40,11 +40,21 @@ class TestSuiteRegtest(TestSuiteBase): tmpdir_nums = [1] color = True deterministic = False - cmd_group = ( - ('setup', 'regtest (Bob and Alice) mode setup'), - ('sendto1', 'sending funds to address #1 (1)'), - ('sendto2', 'sending funds to address #1 (2)'), - ('sendto3', 'sending funds to address #2'), + cmd_group_in = ( + ('setup', 'regtest mode setup'), + ('subgroup.fund_addrbal', []), + ('subgroup.addrbal', ['fund_addrbal']), + ('stop', 'stopping regtest daemon'), + ) + cmd_subgroups = { + 'fund_addrbal': ( + "funding addresses for 'addrbal' subgroup", + ('sendto1', 'sending funds to address #1 (1)'), + ('sendto2', 'sending funds to address #1 (2)'), + ('sendto3', 'sending funds to address #2'), + ), + 'addrbal': ( + "'mmnode-addrbal' script", ('addrbal_single', 'getting address balance (single address)'), ('addrbal_multiple', 'getting address balances (multiple addresses)'), ('addrbal_multiple_tabular1', 'getting address balances (multiple addresses, tabular output)'), @@ -54,8 +64,8 @@ class TestSuiteRegtest(TestSuiteBase): ('addrbal_nobal3', 'getting address balances (one null balance)'), ('addrbal_nobal3_tabular1', 'getting address balances (one null balance, tabular output)'), ('addrbal_nobal3_tabular2', 'getting address balances (one null balance, tabular, show first block)'), - ('stop', 'stopping regtest daemon'), ) + } def __init__(self,trunner,cfgs,spawn): TestSuiteBase.__init__(self,trunner,cfgs,spawn) @@ -88,32 +98,81 @@ class TestSuiteRegtest(TestSuiteBase): def sendto2(self): return self.sendto(self.addrs[0],'0.234') def sendto3(self): return self.sendto(self.addrs[1],'0.345') + def addrbal(self,args,expect_list): + t = self.spawn('mmnode-addrbal',args) + t.match_expect_list(expect_list) + return t + def addrbal_single(self): - return self.spawn('mmnode-addrbal',args2+[self.addrs[0]]) + return self.addrbal( + args2 + [self.addrs[0]], + [ + f'Balance: 0.357 {g.coin}', + '2 unspent outputs in 2 blocks', + '394','0.123', + '395','0.234' + ]) def addrbal_multiple(self): - return self.spawn('mmnode-addrbal',args2+[self.addrs[1],self.addrs[0]]) + return self.addrbal( + args2 + [self.addrs[1],self.addrs[0]], + [ + '396','0.345', + '394','0.123', + '395','0.234' + ]) def addrbal_multiple_tabular1(self): - return self.spawn('mmnode-addrbal',args2+['--tabular',self.addrs[1],self.addrs[0]]) + return self.addrbal( + args2 + ['--tabular',self.addrs[1],self.addrs[0]], + [ + self.addrs[1] + ' 1 396','0.345', + self.addrs[0] + ' 2 395','0.357' + ]) def addrbal_multiple_tabular2(self): - return self.spawn('mmnode-addrbal',args1+['--tabular','--first-block',self.addrs[1],self.addrs[0]]) + return self.addrbal( + args2 + ['--tabular','--first-block',self.addrs[1],self.addrs[0]], + [ + self.addrs[1] + ' 1 396','396','0.345', + self.addrs[0] + ' 2 394','395','0.357' + ]) def addrbal_nobal1(self): - return self.spawn('mmnode-addrbal',args2+[self.addrs[2]]) + return self.addrbal( + args2 + [self.addrs[2]], ['Address has no balance'] ) def addrbal_nobal2(self): - return self.spawn('mmnode-addrbal',args2+[self.addrs[2],self.addrs[3]]) + return self.addrbal( + args2 + [self.addrs[2],self.addrs[3]], ['Addresses have no balances'] ) def addrbal_nobal3(self): - return self.spawn('mmnode-addrbal',args2+[self.addrs[4],self.addrs[0],self.addrs[3]]) + return self.addrbal( + args2 + [self.addrs[4],self.addrs[0],self.addrs[3]], + [ + 'No balance', + '2 unspent outputs in 2 blocks', + '394','0.123','395','0.234', + 'No balance' + ]) def addrbal_nobal3_tabular1(self): - return self.spawn('mmnode-addrbal',args2+['--tabular',self.addrs[4],self.addrs[0],self.addrs[3]]) + return self.addrbal( + args2 + ['--tabular',self.addrs[4],self.addrs[0],self.addrs[3]], + [ + self.addrs[4] + ' - - -', + self.addrs[0] + ' 2 395','0.357', + self.addrs[3] + ' - - -', + ]) def addrbal_nobal3_tabular2(self): - return self.spawn('mmnode-addrbal',args1+['--tabular','--first-block',self.addrs[4],self.addrs[0],self.addrs[3]]) + return self.addrbal( + args2 + ['--tabular','--first-block',self.addrs[4],self.addrs[0],self.addrs[3]], + [ + self.addrs[4] + ' - - - -', + self.addrs[0] + ' 2 394','395','0.357', + self.addrs[3] + ' - - - -', + ]) def stop(self): if opt.no_daemon_stop: From f9a084dba96d090752af663c5f7d0ee155369f50 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 4 Aug 2022 14:16:27 +0000 Subject: [PATCH 007/108] `test.py`: add halving calculator test --- test/test_py_d/ts_regtest.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index a6836be..4fb3eec 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -42,11 +42,21 @@ class TestSuiteRegtest(TestSuiteBase): deterministic = False cmd_group_in = ( ('setup', 'regtest mode setup'), + ('subgroup.halving_calculator', []), ('subgroup.fund_addrbal', []), ('subgroup.addrbal', ['fund_addrbal']), ('stop', 'stopping regtest daemon'), ) cmd_subgroups = { + 'halving_calculator': ( + "'mmnode-halving-calculator' script", + ('halving_calculator1', "halving calculator (--help)"), + ('halving_calculator2', "halving calculator"), + ('halving_calculator3', "halving calculator (--list)"), + ('halving_calculator4', "halving calculator (--mined)"), + ('halving_calculator5', "halving calculator (--mined --bdr-proj=5)"), + ('halving_calculator6', "halving calculator (--mined --sample-size=20)"), + ), 'fund_addrbal': ( "funding addresses for 'addrbal' subgroup", ('sendto1', 'sending funds to address #1 (1)'), @@ -91,6 +101,29 @@ class TestSuiteRegtest(TestSuiteBase): t.expect(s) return t + def halving_calculator(self,add_args,expect_list): + t = self.spawn('mmnode-halving-calculator',args1+add_args) + t.match_expect_list(expect_list) + return t + + def halving_calculator1(self): + return self.halving_calculator(['--help'],['USAGE:']) + + def halving_calculator2(self): + return self.halving_calculator([],['Current block: 393',f'Current block subsidy: 12.5 {g.coin}']) + + def halving_calculator3(self): + return self.halving_calculator(['--list'],['33 4950','0']) + + def halving_calculator4(self): + return self.halving_calculator(['--mined'],['0 0.0000015 14949.9999835']) + + def halving_calculator5(self): + return self.halving_calculator(['--mined','--bdr-proj=5'],['5.00000 0 0.0000015 14949.9999835']) + + def halving_calculator6(self): + return self.halving_calculator(['--mined','--sample-size=20'],['33 4950','0 0.0000015 14949.9999835']) + def sendto(self,addr,amt): return self.spawn('mmgen-regtest',['send',addr,amt]) From e7fcc00b95a9631b26d2e2c9d6cda23103958fe7 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 4 Aug 2022 14:16:28 +0000 Subject: [PATCH 008/108] new script: `mmnode-ticker` Display cryptocurrency and other asset prices in convenient tabular format, with optional display of your portfolio. Output is highly configurable. Usage information: $ mmnode-ticker --help --- cmds/mmnode-ticker | 17 + mmgen_node_tools/Misc.py | 102 +++ mmgen_node_tools/Ticker.py | 777 ++++++++++++++++++++ mmgen_node_tools/Util.py | 19 +- mmgen_node_tools/data/ticker-cfg.yaml | 30 + mmgen_node_tools/data/ticker-portfolio.yaml | 6 + mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 193 +++++ setup.cfg | 3 +- test/ref/ticker/ticker-btc.json | 1 + test/ref/ticker/ticker-cfg.yaml | 24 + test/ref/ticker/ticker-portfolio-bad.yaml | 6 + test/ref/ticker/ticker-portfolio.yaml | 5 + test/ref/ticker/ticker.json | 1 + test/test-release.d/cfg.sh | 12 +- test/test_py_d/cfg.py | 2 + test/test_py_d/ts_misc.py | 223 ++++++ 17 files changed, 1416 insertions(+), 7 deletions(-) create mode 100755 cmds/mmnode-ticker create mode 100755 mmgen_node_tools/Misc.py create mode 100755 mmgen_node_tools/Ticker.py create mode 100644 mmgen_node_tools/data/ticker-cfg.yaml create mode 100644 mmgen_node_tools/data/ticker-portfolio.yaml create mode 100755 mmgen_node_tools/main_ticker.py create mode 100644 test/ref/ticker/ticker-btc.json create mode 100644 test/ref/ticker/ticker-cfg.yaml create mode 100644 test/ref/ticker/ticker-portfolio-bad.yaml create mode 100644 test/ref/ticker/ticker-portfolio.yaml create mode 100644 test/ref/ticker/ticker.json diff --git a/cmds/mmnode-ticker b/cmds/mmnode-ticker new file mode 100755 index 0000000..d6e91fb --- /dev/null +++ b/cmds/mmnode-ticker @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-ticker: Display price information for cryptocurrency and other assets +""" + +from mmgen.main import launch + +launch('ticker',package='mmgen_node_tools') diff --git a/mmgen_node_tools/Misc.py b/mmgen_node_tools/Misc.py new file mode 100755 index 0000000..999b68b --- /dev/null +++ b/mmgen_node_tools/Misc.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmgen_node_tools.Misc: miscellaneous data and functions for the MMGen Node Tools suite +""" + +curl_exit_codes = { + 1: 'Unsupported protocol. This build of curl has no support for this protocol', + 2: 'Failed to initialize', + 3: 'URL malformed. The syntax was not correct', + 4: 'A feature or option that was needed to perform the desired request was not enabled or was explicitly disabled at build-time. To make curl able to do this, you probably need another build of libcurl!', + 5: 'Couldn’t resolve proxy. The given proxy host could not be resolved', + 6: 'Couldn’t resolve host. The given remote host was not resolved', + 7: 'Failed to connect to host', + 8: 'Weird server reply. The server sent data curl couldn’t parse', + 9: 'FTP access denied. The server denied login or denied access to the particular resource or directory you wanted to reach. Most often you tried to change to a directory that doesn’t exist on the server', + 10: 'FTP accept failed. While waiting for the server to connect back when an active FTP session is used, an error code was sent over the control connection or similar', + 11: 'FTP weird PASS reply. Curl couldn’t parse the reply sent to the PASS request', + 12: 'During an active FTP session while waiting for the server to connect back to curl, the timeout expired', + 13: 'FTP weird PASV reply, Curl couldn’t parse the reply sent to the PASV request', + 14: 'FTP weird 227 format. Curl couldn’t parse the 227-line the server sent', + 15: 'FTP can’t get host. Couldn’t resolve the host IP we got in the 227-line', + 16: 'HTTP/2 error. A problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems, see the error message for details', + 17: 'FTP couldn’t set binary. Couldn’t change transfer method to binary', + 18: 'Partial file. Only a part of the file was transferred', + 19: 'FTP couldn’t download/access the given file, the RETR (or similar) command failed', + 21: 'FTP quote error. A quote command returned error from the server', + 22: 'HTTP page not retrieved. The requested url was not found or returned another error with the HTTP error code being 400 or above. This return code only appears if -f, --fail is used', + 23: 'Write error. Curl couldn’t write data to a local filesystem or similar', + 25: 'FTP couldn’t STOR file. The server denied the STOR operation, used for FTP uploading', + 26: 'Read error. Various reading problems', + 27: 'Out of memory. A memory allocation request failed', + 28: 'Operation timeout. The specified time-out period was reached according to the conditions', + 30: 'FTP PORT failed. The PORT command failed. Not all FTP servers support the PORT command, try doing a transfer using PASV instead!', + 31: 'FTP couldn’t use REST. The REST command failed. This command is used for resumed FTP transfers', + 33: 'HTTP range error. The range "command" didn’t work', + 34: 'HTTP post error. Internal post-request generation error', + 35: 'SSL connect error. The SSL handshaking failed', + 36: 'Bad download resume. Couldn’t continue an earlier aborted download', + 37: 'FILE couldn’t read file. Failed to open the file. Permissions?', + 38: 'LDAP cannot bind. LDAP bind operation failed', + 39: 'LDAP search failed', + 41: 'Function not found. A required LDAP function was not found', + 42: 'Aborted by callback. An application told curl to abort the operation', + 43: 'Internal error. A function was called with a bad parameter', + 45: 'Interface error. A specified outgoing interface could not be used', + 47: 'Too many redirects. When following redirects, curl hit the maximum amount', + 48: 'Unknown option specified to libcurl. This indicates that you passed a weird option to curl that was passed on to libcurl and rejected. Read up in the manual!', + 49: 'Malformed telnet option', + 51: 'The peer’s SSL certificate or SSH MD5 fingerprint was not OK', + 52: 'The server didn’t reply anything, which here is considered an error', + 53: 'SSL crypto engine not found', + 54: 'Cannot set SSL crypto engine as default', + 55: 'Failed sending network data', + 56: 'Failure in receiving network data', + 58: 'Problem with the local certificate', + 59: 'Couldn’t use specified SSL cipher', + 60: 'Peer certificate cannot be authenticated with known CA certificates', + 61: 'Unrecognized transfer encoding', + 62: 'Invalid LDAP URL', + 63: 'Maximum file size exceeded', + 64: 'Requested FTP SSL level failed', + 65: 'Sending the data requires a rewind that failed', + 66: 'Failed to initialise SSL Engine', + 67: 'The user name, password, or similar was not accepted and curl failed to log in', + 68: 'File not found on TFTP server', + 69: 'Permission problem on TFTP server', + 70: 'Out of disk space on TFTP server', + 71: 'Illegal TFTP operation', + 72: 'Unknown TFTP transfer ID', + 73: 'File already exists (TFTP)', + 74: 'No such user (TFTP)', + 75: 'Character conversion failed', + 76: 'Character conversion functions required', + 77: 'Problem with reading the SSL CA cert (path? access rights?)', + 78: 'The resource referenced in the URL does not exist', + 79: 'An unspecified error occurred during the SSH session', + 80: 'Failed to shut down the SSL connection', + 82: 'Could not load CRL file, missing or wrong format (added in 7.19.0)', + 83: 'Issuer check failed (added in 7.19.0)', + 84: 'The FTP PRET command failed', + 85: 'RTSP: mismatch of CSeq numbers', + 86: 'RTSP: mismatch of Session Identifiers', + 87: 'unable to parse FTP file list', + 88: 'FTP chunk callback reported error', + 89: 'No connection available, the session will be queued', + 90: 'SSL public key does not matched pinned public key', + 91: 'Invalid SSL certificate status', + 92: 'Stream error in HTTP/2 framing layer', + 93: 'An API function was called from inside a callback', + 94: 'An authentication function returned an error', + 95: 'A problem was detected in the HTTP/3 layer. This is somewhat generic and can be one out of several problems, see the error message for details', + 96: 'QUIC connection error. This error may be caused by an SSL library error. QUIC is the protocol used for HTTP/3 transfers', +} diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py new file mode 100755 index 0000000..cafe815 --- /dev/null +++ b/mmgen_node_tools/Ticker.py @@ -0,0 +1,777 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets +""" + +api_host = 'api.coinpaprika.com' +api_url = f'https://{api_host}/v1/ticker' +ratelimit = 240 +btc_ratelimit = 10 + +# We use deprecated coinpaprika ‘ticker’ API for now because it returns ~45% less data. +# Old ‘ticker’ API (/v1/ticker): data['BTC']['price_usd'] +# New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price'] + +# Possible alternatives: +# - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR + +import sys,os,time,json,yaml +from subprocess import run,PIPE,CalledProcessError +from decimal import Decimal +from collections import namedtuple +from mmgen.opts import opt +from mmgen.globalvars import g +from mmgen.color import * +from mmgen.util import die,fmt_list,msg,msg_r,Msg,do_pager,suf,fmt + +homedir = os.getenv('HOME') +cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') +cfg_fn = 'ticker-cfg.yaml' +portfolio_fn = 'ticker-portfolio.yaml' + +def assets_list_gen(cfg_in): + for k,v in cfg_in.cfg['assets'].items(): + yield('') + yield(k.upper()) + for e in v: + yield(' {:4s} {}'.format(*e.split('-',1))) + +def gen_data(data): + """ + Filter the raw data and return it as a dict keyed by the IDs of the assets + we want to display. + + Add dummy entry for USD and entry for user-specified asset, if any. + + Since symbols in source data are not guaranteed to be unique (e.g. XAG), we + must search the data twice: first for unique IDs, then for symbols while + checking for duplicates. + """ + + def dup_sym_errmsg(dup_sym): + return ( + f'The symbol {dup_sym!r} is shared by the following assets:\n' + + '\n ' + '\n '.join(d['id'] for d in data if d['symbol'] == dup_sym) + + '\n\nPlease specify the asset by one of the full IDs listed above\n' + + f'instead of {dup_sym!r}' + ) + + def check_assets_found(wants,found,keys=['symbol','id']): + error = False + for k in keys: + missing = wants[k] - found[k] + if missing: + msg( + ('The following IDs were not found in source data:\n{}' if k == 'id' else + 'The following symbols could not be resolved:\n{}').format( + fmt_list(missing,fmt='col',indent=' ') + )) + error = True + if error: + die(1,'Missing data, exiting') + + rows_want = { + 'id': {r.id for r in cfg.rows if getattr(r,'id',None)} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, + } + usr_assets = cfg.usr_rows + cfg.usr_columns + tuple(c for c in (cfg.query or ()) if c) + usr_wants = { + 'id': ( + {a.id for a in usr_assets if a.id} - + {a.id for a in usr_assets if a.amount and a.id} - {'usd-us-dollar'} ) + , + 'symbol': ( + {a.symbol for a in usr_assets if not a.id} - + {a.symbol for a in usr_assets if a.amount} - {'USD'} ), + } + + found = { 'id': set(), 'symbol': set() } + + for k in ['id','symbol']: + wants = rows_want[k] | usr_wants[k] + if wants: + for d in data: + if d[k] in wants: + if d[k] in found[k]: + die(1,dup_sym_errmsg(d[k])) + yield (d['id'],d) + found[k].add(d[k]) + if k == 'id' and len(found[k]) == len(wants): + break + + for d in data: + if d['id'] == 'btc-bitcoin': + btcusd = Decimal(d['price_usd']) + break + + for asset in (cfg.usr_rows + cfg.usr_columns): + if asset.amount: + """ + User-supplied rate overrides rate from source data. + """ + _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() + yield ( _id, { + 'symbol': asset.symbol, + 'id': _id, + 'price_usd': str(Decimal(1/asset.amount)), + 'price_btc': str(Decimal(1/asset.amount/btcusd)), + 'last_updated': int(now), + }) + + yield ('usd-us-dollar', { + 'symbol': 'USD', + 'id': 'usd-us-dollar', + 'price_usd': '1.0', + 'price_btc': str(Decimal(1/btcusd)), + 'last_updated': int(now), + }) + + check_assets_found(usr_wants,found) + +def get_src_data(curl_cmd): + + tor_captcha_msg = f""" + If you’re using Tor, the API request may have failed due to Captcha protection. + A workaround for this issue is to retrieve the JSON data with a browser from + the following URL: + + {api_url} + + and save it to: + + ‘{cfg.cachedir}/ticker.json’ + + Then invoke the program with --cached-data and without --btc + """ + + def rate_limit_errmsg(timeout,elapsed): + return ( + f'Rate limit exceeded! Retry in {timeout-elapsed} seconds' + + ('' if cfg.btc_only else ', or use --cached-data or --btc') + ) + + if not os.path.exists(cachedir): + os.makedirs(cachedir) + + if cfg.btc_only: + fn = os.path.join(cfg.cachedir,'ticker-btc.json') + timeout = 5 if g.test_suite else btc_ratelimit + else: + fn = os.path.join(cfg.cachedir,'ticker.json') + timeout = 5 if g.test_suite else ratelimit + + fn_rel = os.path.relpath(fn,start=homedir) + + if not os.path.exists(fn): + open(fn,'w').write('{}') + + if opt.cached_data: + json_text = open(fn).read() + else: + elapsed = int(time.time() - os.stat(fn).st_mtime) + if elapsed >= timeout: + msg_r(f'Fetching data from {api_host}...') + try: + cp = run(curl_cmd,check=True,stdout=PIPE) + except CalledProcessError as e: + msg('') + from .Misc import curl_exit_codes + msg(red(curl_exit_codes[e.returncode])) + msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) + from mmgen.exception import MMGenCalledProcessError + raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') + json_text = cp.stdout.decode() + msg('done') + else: + die(1,rate_limit_errmsg(timeout,elapsed)) + + try: + data = json.loads(json_text) + except: + msg(json_text[:1024] + '...') + msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) + die(2,'Retrieved data is not valid JSON, exiting') + + if not data: + if opt.cached_data: + die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') + else: + die(2,'Remote host returned no data!') + elif 'error' in data: + die(1,data['error']) + + if opt.cached_data: + msg(f'Using cached data from ~/{fn_rel}') + else: + open(fn,'w').write(json_text) + msg(f'JSON data cached to ~/{fn_rel}') + + return data + +def main(cfg_parm,cfg_in_parm): + + def update_sample_file(usr_cfg_file): + src_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() + sample_file = usr_cfg_file + '.sample' + sample_data = open(sample_file).read() if os.path.exists(sample_file) else None + if src_data != sample_data: + os.makedirs(os.path.dirname(sample_file),exist_ok=True) + msg('{} {}'.format( + ('Updating','Creating')[sample_data is None], + sample_file )) + open(sample_file,'w').write(src_data) + + def get_curl_cmd(): + return ([ + 'curl', + '--tr-encoding', + '--compressed', # adds 'Accept-Encoding: gzip' + '--silent', + '--header', 'Accept: application/json', + ] + + (['--proxy', cfg.proxy] if cfg.proxy else []) + + [api_url + ('/btc-bitcoin' if cfg.btc_only else '')] + ) + + global cfg,cfg_in + cfg = cfg_parm + cfg_in = cfg_in_parm + + try: + from importlib.resources import files # Python 3.9 + except ImportError: + from importlib_resources import files + + update_sample_file(cfg_in.cfg_file) + update_sample_file(cfg_in.portfolio_file) + + if opt.portfolio and not cfg_in.portfolio: + die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( + os.path.relpath(cfg_in.portfolio_file,start=homedir))) + + curl_cmd = get_curl_cmd() + + if opt.print_curl: + Msg(curl_cmd + '\n' + ' '.join(curl_cmd)) + return + + parsed_json = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) + + if opt.list_ids: + do_pager('\n'.join(e['id'] for e in parsed_json)) + return + + global now + now = 1659465400 if g.test_suite else time.time() # 1659524400 1659445900 + + (do_pager if opt.pager else Msg)( + '\n'.join(getattr(Ticker,cfg.clsname)(dict(gen_data(parsed_json))).gen_output()) + ) + +def make_cfg(cmd_args,cfg_in): + + def get_rows_from_cfg(add_data=None): + def create_row(e): + return asset_tuple(e.split('-')[0].upper(),e) + def gen(): + for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): + yield(k) + if add_data and k in add_data: + v += tuple(add_data[k]) + for e in v: + yield(create_row(e)) + return tuple(gen()) + + def parse_asset_tuple(s): + sym,id = (s.split('-')[0],s) if '-' in s else (s,None) + return asset_tuple( sym.upper(), id.lower() if id else None ) + + def parse_asset_triplet(s): + ss = s.split(':') + sym,amt = ( ss[0], Decimal(ss[1]) ) if len(ss) == 2 else ( s, None ) + return asset_triplet( *parse_asset_tuple(sym), amt ) + + def parse_usr_asset_arg(s): + return tuple(parse_asset_triplet(ss) for ss in s.split(',')) if s else () + + def parse_query_arg(s): + ss = s.split(':') + if len(ss) == 2: + return query_tuple( + asset = parse_asset_triplet(s), + to_asset = None ) + elif len(ss) in (3,4): + return query_tuple( + asset = parse_asset_triplet(':'.join(ss[:2])), + to_asset = parse_asset_triplet(':'.join(ss[2:])), + ) + else: + die(1,f'{s}: malformed argument') + + def gen_uniq(obj_list,key,preload=None): + found = set([getattr(obj,key) for obj in preload if hasattr(obj,key)] if preload else ()) + for obj in obj_list: + id = getattr(obj,key) + if id not in found: + yield obj + found.add(id) + + def get_usr_assets(): + return ( + 'user_added', + usr_rows + + (tuple(asset for asset in query if asset) if query else ()) + + usr_columns ) + + def get_portfolio_assets(ret=()): + if cfg_in.portfolio and opt.portfolio: + ret = tuple( asset_tuple(e.split('-')[0].upper(),e) for e in cfg_in.portfolio ) + return ( 'portfolio', tuple(e for e in ret if (not opt.btc) or e.symbol == 'BTC') ) + + def get_portfolio(): + return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not opt.btc) or k == 'btc-bitcoin'} + + def parse_add_precision(s): + if not s: + return 0 + if not (s.isdigit() and s.isascii()): + die(1,f'{s}: invalid parameter for --add-precision (not an integer)') + if int(s) > 30: + die(1,f'{s}: invalid parameter for --add-precision (value >30)') + return int(s) + + def create_rows(): + rows = ( + ('trade_pair',) + query if (query and query.to_asset) else + ('bitcoin',parse_asset_tuple('btc-bitcoin')) if opt.btc else + get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if opt.add_columns else None ) + ) + + for hdr,data in ( + (get_usr_assets(),) if query else + (get_usr_assets(), get_portfolio_assets()) + ): + if data: + uniq_data = tuple(gen_uniq(data,'symbol',preload=rows)) + if uniq_data: + rows += (hdr,) + uniq_data + return rows + + cfg_tuple = namedtuple('global_cfg',[ + 'rows', + 'usr_rows', + 'usr_columns', + 'query', + 'adjust', + 'clsname', + 'btc_only', + 'add_prec', + 'cachedir', + 'proxy', + 'portfolio' ]) + + query_tuple = namedtuple('query',['asset','to_asset']) + asset_triplet = namedtuple('asset_triplet',['symbol','id','amount']) + asset_tuple = namedtuple('asset_tuple',['symbol','id']) + + usr_rows = parse_usr_asset_arg(opt.add_rows) + usr_columns = parse_usr_asset_arg(opt.add_columns) + query = parse_query_arg(cmd_args[0]) if cmd_args else None + + return cfg_tuple( + rows = create_rows(), + usr_rows = usr_rows, + usr_columns = usr_columns, + query = query, + adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(opt.adjust or 0) ), + clsname = 'trading' if query else 'overview', + btc_only = opt.btc, + add_prec = parse_add_precision(opt.add_precision), + cachedir = opt.cachedir or cfg_in.cfg.get('cachedir') or cachedir, + proxy = None if opt.proxy == '' else (opt.proxy or cfg_in.cfg.get('proxy')), + portfolio = get_portfolio() if cfg_in.portfolio and opt.portfolio and not query else None + ) + +def get_cfg_in(): + ret = namedtuple('cfg_in_data',['cfg','portfolio','cfg_file','portfolio_file']) + cfg_file,portfolio_file = ( + [os.path.join(g.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)] + ) + cfg_data,portfolio_data = ( + [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None for fn in (cfg_file,portfolio_file)] + ) + return ret( + cfg = cfg_data or { + 'assets': { + 'coin': [ 'btc-bitcoin', 'eth-ethereum', 'xmr-monero' ], + 'commodity': [ 'xau-gold-spot-token', 'xag-silver-spot-token', 'xbr-brent-crude-oil-spot' ], + 'fiat': [ 'gbp-pound-sterling-token', 'eur-euro-token' ], + 'index': [ 'dj30-dow-jones-30-token', 'spx-sp-500', 'ndx-nasdaq-100-token' ], + }, + 'proxy': 'http://vpn-gw:8118' + }, + portfolio = portfolio_data, + cfg_file = cfg_file, + portfolio_file = portfolio_file, + ) + +class Ticker: + + class base: + + offer = None + to_asset = None + + def __init__(self,data): + + self.comma = ',' if opt.thousands_comma else '' + + self.col1_wid = max(len('TOTAL'),( + max(len(self.create_label(d['id'])) for d in data.values()) if opt.name_labels else + max(len(d['symbol']) for d in data.values()) + )) + 1 + + self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows] + self.col_usd_prices = {k:Decimal(self.data[k]['price_usd']) for k in self.col_ids} + + self.prices = {row.id:self.get_row_prices(row.id) + for row in self.rows if isinstance(row,tuple) and row.id in data} + self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') + + def format_last_update_col(self,cross_assets=()): + + if opt.elapsed: + from .Util import format_elapsed_hr + fmt_func = format_elapsed_hr + else: + fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) # ticker API + # t.replace('T',' ').replace('Z','') # tickers API + + d = self.data + max_w = 0 + min_t = min( (int(d[a.id]['last_updated']) for a in cross_assets), default=None ) + + for row in self.rows: + if isinstance(row,tuple): + try: + t = int(d[row.id]['last_updated']) + except KeyError: + pass + else: + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now ) + max_w = max(len(t_fmt),max_w) + + self.upd_w = max_w + + def init_prec(self): + exp = [(a.id,Decimal.adjusted(self.prices[a.id]['usd-us-dollar'])) for a in self.usr_col_assets] + self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp } + self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp } + + def get_id(self,asset): + if asset.id: + return asset.id + else: + for d in self.data.values(): + if d['symbol'] == asset.symbol: + return d['id'] + + def create_label(self,id): + return ' '.join(id.split('-')[1:]).upper() + + def gen_output(self): + yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now))) + + for asset in self.usr_col_assets: + if asset.symbol != 'USD': + usdprice = Decimal(self.data[asset.id]['price_usd']) + yield '{} ({}) = {:{}.{}f} USD'.format( + asset.symbol, + self.create_label(asset.id), + usdprice, + self.comma, + max(2,int(-usdprice.adjusted())+4) ) + + if hasattr(self,'subhdr'): + yield self.subhdr + + if self.show_adj: + yield ( + ('Offered price differs from spot' if self.offer else 'Adjusting prices') + + ' by ' + + yellow('{:+.2f}%'.format( (self.adjust-1) * 100 )) + ) + + yield '' + + if cfg.portfolio: + yield blue('PRICES') + + if self.table_hdr: + yield self.table_hdr + + for row in self.rows: + if isinstance(row,str): + yield ('-' * self.hl_wid) + else: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') + + yield '-' * self.hl_wid + + if cfg.portfolio: + self.fs_num = self.fs_num2 + self.fs_str = self.fs_str2 + yield '' + yield blue('PORTFOLIO') + yield self.table_hdr + yield '-' * self.hl_wid + for sym,amt in cfg.portfolio.items(): + try: + yield self.fmt_row(self.data[sym],amt=amt) + except KeyError: + yield gray(f'(no data for {sym})') + yield '-' * self.hl_wid + if not cfg.btc_only: + yield self.fs_num.format( + lbl = 'TOTAL', pc1='', pc2='', upd='', amt='', + **{ k.replace('-','_'): v for k,v in self.prices['total'].items() } + ) + + class overview(base): + + def __init__(self,data): + self.data = data + self.adjust = cfg.adjust + self.show_adj = self.adjust != 1 + self.usr_col_assets = [asset._replace(id=self.get_id(asset)) for asset in cfg.usr_columns] + self.col_ids = ('usd-us-dollar',) + tuple(a.id for a in self.usr_col_assets) + ('btc-bitcoin',) + + super().__init__(data) + + self.format_last_update_col() + + if cfg.portfolio: + self.prices['total'] = { col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + for row in self.rows if isinstance(row,tuple) and row.id in cfg.portfolio and row.id in data) + for col_id in self.col_ids } + + self.init_prec() + self.init_fs() + + def get_row_prices(self,id): + if id in self.data: + d = self.data[id] + return { k: ( + Decimal(d['price_btc']) if k == 'btc-bitcoin' else + Decimal(d['price_usd']) / self.col_usd_prices[k] + ) * self.adjust for k in self.col_ids } + + def fmt_row(self,d,amt=None,amt_fmt=None): + + def fmt_pct(d): + if d in ('',None): + return gray(' --') + n = Decimal(d) + return (red,green)[n>=0](f'{n:+7.2f}') + + p = self.prices[d['id']] + + if amt is not None: + amt_fmt = f'{amt:{19+cfg.add_prec}{self.comma}.{8+cfg.add_prec}f}' + if '.' in amt_fmt: + amt_fmt = amt_fmt.rstrip('0').rstrip('.') + + return self.fs_num.format( + lbl = (self.create_label(d['id']) if opt.name_labels else d['symbol']), + pc1 = fmt_pct(d.get('percent_change_7d')), + pc2 = fmt_pct(d.get('percent_change_24h')), + upd = d.get('last_updated_fmt'), + amt = amt_fmt, + **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() } + ) + + def init_fs(self): + + col_prec = {'usd-us-dollar':2+cfg.add_prec,'btc-bitcoin':8+cfg.add_prec } # | self.uprec # Python 3.9 + col_prec.update(self.uprec) + col_wid = {'usd-us-dollar':8+cfg.add_prec,'btc-bitcoin':12+cfg.add_prec } # """ + col_wid.update(self.uwid) + max_row = max( + ( (k,v['btc-bitcoin']) for k,v in self.prices.items() ), + key = lambda a: a[1] + ) + widths = { k: len('{:{}.{}f}'.format( self.prices[max_row[0]][k], self.comma, col_prec[k] )) + for k in self.col_ids } + + fd = namedtuple('format_str_data',['fs_str','fs_num','wid']) + + col_fs_data = { + 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid), + 'pct7d': fd(' {pc1:7}', ' {pc1:7}', 8), + 'pct24h': fd(' {pc2:7}', ' {pc2:7}', 8), + 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2), + 'amt': fd(' {amt}', ' {amt}', 21), + } +# } | { k: fd( # Python 3.9 + col_fs_data.update({ k: fd( + ' {{{}:>{}}}'.format( k.replace('-','_'), widths[k] ), + ' {{{}:{}{}.{}f}}'.format( k.replace('-','_'), widths[k], self.comma, col_prec[k] ), + widths[k]+2 + ) for k in self.col_ids + }) + + cols = ( + ['label','usd-us-dollar'] + + [asset.id for asset in self.usr_col_assets] + + [a for a,b in ( + ( 'btc-bitcoin', not cfg.btc_only ), + ( 'pct7d', opt.percent_change ), + ( 'pct24h', opt.percent_change ), + ( 'update_time', opt.update_time ), + ) if b] + ) + cols2 = list(cols) + if opt.update_time: + cols2.pop() + cols2.append('amt') + + self.fs_str = ''.join(col_fs_data[c].fs_str for c in cols) + self.fs_num = ''.join(col_fs_data[c].fs_num for c in cols) + self.hl_wid = sum(col_fs_data[c].wid for c in cols) + + self.fs_str2 = ''.join(col_fs_data[c].fs_str for c in cols2) + self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) + self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) + + @property + def table_hdr(self): + return self.fs_str.format( + lbl = '', + pc1 = ' CHG_7d', + pc2 = 'CHG_24h', + upd = 'UPDATED', + amt = ' AMOUNT', + usd_us_dollar = 'USD', + btc_bitcoin = ' BTC', + **{ a.id.replace('-','_'): a.symbol for a in self.usr_col_assets } + ) + + class trading(base): + + def __init__(self,data): + self.data = data + self.asset = cfg.query.asset._replace(id=self.get_id(cfg.query.asset)) + self.to_asset = ( + cfg.query.to_asset._replace(id=self.get_id(cfg.query.to_asset)) + if cfg.query.to_asset else None ) + self.col_ids = [self.asset.id] + self.adjust = cfg.adjust + if self.to_asset: + self.offer = self.to_asset.amount + if self.offer: + real_price = ( + self.asset.amount + * Decimal(data[self.asset.id]['price_usd']) + / Decimal(data[self.to_asset.id]['price_usd']) + ) + if self.adjust != 1: + die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier') + self.adjust = self.offer / real_price + self.hl_ids = [self.asset.id,self.to_asset.id] + else: + self.hl_ids = [self.asset.id] + + self.show_adj = self.adjust != 1 or self.offer + + super().__init__(data) + + self.usr_col_assets = [self.asset] + ([self.to_asset] if self.to_asset else []) + for a in self.usr_col_assets: + self.prices[a.id]['usd-us-dollar'] = Decimal(data[a.id]['price_usd']) + + self.format_last_update_col(cross_assets=self.usr_col_assets) + + self.init_prec() + self.init_fs() + + def get_row_prices(self,id): + if id in self.data: + d = self.data[id] + return { k: self.col_usd_prices[self.asset.id] / Decimal(d['price_usd']) for k in self.col_ids } + + def init_fs(self): + self.max_wid = max( + len('{:{}{}.{}f}'.format( + v[self.asset.id] * self.asset.amount, + 16 + cfg.add_prec, + self.comma, + 8 + cfg.add_prec + )) + for v in self.prices.values() + ) + self.fs_str = '{lbl:%s} {p_spot}' % self.col1_wid + self.hl_wid = self.col1_wid + self.max_wid + 1 + if self.show_adj: + self.fs_str += ' {p_adj}' + self.hl_wid += self.max_wid + 1 + if opt.update_time: + self.fs_str += ' {upd}' + self.hl_wid += self.upd_w + 2 + + def fmt_row(self,d): + id = d['id'] + p = self.prices[id][self.asset.id] * self.asset.amount + p_spot = '{:{}{}.{}f}'.format( p, self.max_wid, self.comma, 8+cfg.add_prec ) + p_adj = ( + '{:{}{}.{}f}'.format( p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec ) + if self.show_adj else '' ) + + return self.fs_str.format( + lbl = (self.create_label(id) if opt.name_labels else d['symbol']), + p_spot = green(p_spot) if id in self.hl_ids else p_spot, + p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, + upd = d.get('last_updated_fmt'), + ) + + @property + def table_hdr(self): + return self.fs_str.format( + lbl = '', + p_spot = '{t:>{w}}'.format( + t = 'SPOT PRICE', + w = self.max_wid ), + p_adj = '{t:>{w}}'.format( + t = ('OFFERED' if self.offer else 'ADJUSTED') + ' PRICE', + w = self.max_wid ), + upd = 'UPDATED' + ) + + @property + def subhdr(self): + return ( + '{a}: {b:{c}} {d}'.format( + a = 'Offer' if self.offer else 'Amount', + b = self.asset.amount, + c = self.comma, + d = self.asset.symbol + ) + ( + ( + ' =>' + + (' {:{}}'.format(self.offer,self.comma) if self.offer else '') + + ' {} ({})'.format( + self.to_asset.symbol, + self.create_label(self.to_asset.id) ) + ) if self.to_asset else '' ) + ) diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 2a67fe2..7fdc409 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -19,8 +19,8 @@ mmgen_node_tools.Util: utility functions for MMGen node tools """ -import time,subprocess -from mmgen.util import msg +import time +from mmgen.util import suf def get_hms(t=None,utc=False,no_secs=False): secs = t or time.time() @@ -33,11 +33,26 @@ def get_day_hms(t=None,utc=False): ret = (time.localtime,time.gmtime)[utc](secs) return '{:04}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*ret[0:6]) +def format_elapsed_hr(t,now=None,cached={}): + now = now or time.time() + e = int(now - t) + if not e in cached: + h = abs(e) // 3600 + m = abs(e) // 60 % 60 + cached[e] = '{a}{b} minute{c} {d}'.format( + a = '{} hour{}, '.format(h,suf(h)) if h else '', + b = m, + c = suf(m), + d = 'ago' if e > 0 else 'in the future' ) if (h or m) else 'just now' + return cached[e] + def do_system(cmd,testing=False,shell=False): if testing: + from mmgen.util import msg msg("Would execute: '%s'" % cmd) return True else: + import subprocess return subprocess.call((cmd if shell else cmd.split()),shell,stderr=subprocess.PIPE) def get_url(url,gzip_ok=False,proxy=None,timeout=60,verbose=False,debug=False): diff --git a/mmgen_node_tools/data/ticker-cfg.yaml b/mmgen_node_tools/data/ticker-cfg.yaml new file mode 100644 index 0000000..2566da4 --- /dev/null +++ b/mmgen_node_tools/data/ticker-cfg.yaml @@ -0,0 +1,30 @@ +# Indentation must be consistent! Do not mix leading tabs and spaces. + +# See the curl manpage for supported --proxy parameters +# For a direct connection, leave the right-hand side blank +proxy: http://vpn-gw:8118 + +# Override the default cache directory (~/.cache/mmgen-node-tools) +cachedir: + +# Asset labels are arbitrary strings. Use as many or few as you wish. +# Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. +assets: + coin1: + - btc-bitcoin + - eth-ethereum + - xmr-monero + coin2: + - ada-cardano + - bnb-binance-coin + commodity: + - xau-gold-spot-token + - xag-silver-spot-token + - xbr-brent-crude-oil-spot + fiat: + - gbp-pound-sterling-token + - eur-euro-token + index: + - dj30-dow-jones-30-token + - spx-sp-500 + - ndx-nasdaq-100-token diff --git a/mmgen_node_tools/data/ticker-portfolio.yaml b/mmgen_node_tools/data/ticker-portfolio.yaml new file mode 100644 index 0000000..fcadd30 --- /dev/null +++ b/mmgen_node_tools/data/ticker-portfolio.yaml @@ -0,0 +1,6 @@ +# List each asset and amount in your portfolio here. +# Amounts should be strings rather than floating-point values. +# Invoke `mmnode-ticker --list-ids` for a full list of supported asset IDs. +btc-bitcoin: '1.23456789' +eth-ethereum: '2.3456789012' +xmr-monero: '4.5678901234' diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 59afffe..7fc0f94 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev3 +3.1.dev4 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py new file mode 100755 index 0000000..1949a66 --- /dev/null +++ b/mmgen_node_tools/main_ticker.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools + +""" +mmnode-ticker: Display price information for cryptocurrency and other assets +""" + +import sys,os +from mmgen.common import * +from .Ticker import * + +opts_data = { + 'sets': [ + ('wide', True, 'percent_change', True), + ('wide', True, 'name_labels', True), + ('wide', True, 'thousands_comma', True), + ('wide', True, 'update_time', True), + ], + 'text': { + 'desc': 'Display prices for cryptocurrency and other assets', + 'usage': '[opts] [TRADE_SPECIFIER]', + 'options': f""" +-h, --help Print this help message +--, --longhelp Print help message for long options (common options) +-A, --adjust=P Adjust prices by percentage ‘P’. In ‘trading’ mode, + spot and adjusted prices are shown in separate columns. +-b, --btc Fetch and display data for Bitcoin only +-c, --add-columns=LIST Add columns for asset specifiers in LIST (comma- + separated, see ASSET SPECIFIERS below). Can also be + used to supply a USD exchange rate for missing assets. +-C, --cached-data Use cached data from previous network query instead of + live data from server +-d, --cachedir=D Read and write cached JSON data to directory ‘D’ + instead of ‘~/{os.path.relpath(cachedir,start=homedir)}’ +-e, --add-precision=N Add ‘N’ digits of precision to columns +-E, --elapsed Show elapsed time in UPDATED column (see --update-time) +-F, --portfolio Display portfolio data +-l, --list-ids List IDs of all available assets +-n, --name-labels Label rows with asset names rather than symbols +-p, --percent-change Add percentage change columns +-P, --pager Pipe the output to a pager +-r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, + see ASSET SPECIFIERS below). Can also be used to supply + a USD exchange rate for missing assets. +-T, --thousands-comma Use comma as a thousands separator +-u, --update-time Include UPDATED (last update time) column +-U, --print-curl Print cURL command to standard output and exit +-w, --wide Display all optional columns (equivalent to -punT) +-x, --proxy=P Connect via proxy ‘P’. Set to the empty string to + disable. Consult the curl manpage for --proxy usage. +""", + 'notes': """ + +The script has two display modes: ‘overview’, the default, and ‘trading’, the +latter being enabled when a TRADE_SPECIFIER argument (see below) is supplied +on the command line. + +Overview mode displays prices of all configured assets, and optionally the +user’s portfolio, while trading mode displays the price of a given quantity +of an asset in relation to other assets, optionally comparing an offered +price to the spot price. + +ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID consisting of +symbol plus label (e.g. ‘xmr-monero’). In cases where the symbol is +ambiguous, the full ID must be used. Examples: + + chf - specify asset by symbol + chf-swiss-franc-token - same as above, but use full ID instead of symbol + +ASSET SPECIFIERS consist of an ASSET followed by an optional colon and USD +rate. If the asset is not in the source data (see --list-ids), the label +part of the ID may be arbitrarily chosen by the user. Examples: + + inr:79.5 - INR is not in the source data, so supply USD rate + inr-indian-rupee:79.5 - same as above, but add an arbitrary label + ada-cardano:0.51 - ADA is in the source data, so use the listed ID + +A TRADE_SPECIFIER is a single argument in the format: + + ASSET:AMOUNT[:TO_ASSET[:TO_AMOUNT]] + + Examples: + + xmr:17.34 - price of 17.34 XMR in all configured assets + xmr-monero:17.34 - same as above, but with full ID + xmr:17.34:eur - price of 17.34 XMR in EUR only + xmr:17.34:eur:2800 - commission on an offer of 17.34 XMR for 2800 EUR + + TO_AMOUNT, if included, is used to calculate the percentage difference or + commission on an offer compared to the spot price. + + If either ASSET or TO_ASSET refer to assets not present in the source data, + a USD rate for the missing asset(s) must be supplied via the --add-columns + or --add-rows options. + + + PROXY NOTE + +The remote server used to obtain the price data, {api_host!r}, blocks +Tor behind a Captcha wall, so a Tor proxy cannot be used directly. If you’re +concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then +set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’ +option in the config file or --proxy on the command line accordingly. Or run +the script directly on the VPN’ed host with ’proxy’ or --proxy set to the +null string. + +Alternatively, you may download the JSON source data in a Tor-proxied browser +from ‘{api_url}’, save it as ‘ticker.json’ in your +configured cache directory, and run the script with the --cached-data option. + + + RATE LIMITING NOTE + +To protect user privacy, all filtering and processing of data is performed +client side so that the remote server does not know which assets are being +examined. This means that data for ALL available assets (currently over 4000) +is fetched with each invocation of the script. A rate limit of {ratelimit} seconds +between calls is thus imposed to prevent abuse of the remote server. When the +--btc option is in effect, this limit is reduced to 10 seconds. To bypass the +rate limit entirely, use --cached-data. + + + EXAMPLES + +# Basic display in ‘overview’ mode: +$ mmnode-ticker + +# Display BTC price only: +$ mmnode-ticker --btc + +# Wide display, add EUR and INR columns, INR rate, extra precision and proxy: +$ mmnode-ticker -w -c eur,inr-indian-rupee:79.5 -e2 -x http://vpnhost:8118 + +# Wide display, use cached data from previous network query, show portfolio +# (see above), pipe output to pager, add DOGE row: +$ mmnode-ticker -wCFP -r doge + +# Display 17.234 XMR priced in all configured assets (‘trading’ mode): +$ mmnode-ticker xmr:17.234 + +# Same as above, but add INR price at specified USDINR rate: +$ mmnode-ticker -c inr:79.5 xmr:17.234 + +# Same as above, but view INR price only at specified rate, adding label: +$ mmnode-ticker -c inr-indian-rupee:79.5 xmr:17.234:inr + +# Calculate commission on an offer of 2700 USD for 0.123 BTC, compared to +# current spot price: +$ mmnode-ticker usd:2700:btc:0.123 + +# Calculate commission on an offer of 200000 INR for 0.1 BTC, compared to +# current spot price, at specified USDINR rate: +$ mmnode-ticker -n -c inr-indian-rupee:79.5 inr:200000:btc:0.1 + + +CONFIGURED ASSETS: +{assets} + +Customize output by editing the file + ~/{cfg} + +To add a portfolio, edit the file + ~/{pf_cfg} +""" + }, + 'code': { + 'notes': lambda s: s.format( + assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), + cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), + pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + api_host = api_host, + api_url = api_url, + ratelimit = ratelimit, + ) + } +} + +cmd_args = opts.init(opts_data,do_post_init=True) + +cfg_in = get_cfg_in() + +cfg = make_cfg(cmd_args,cfg_in) + +opts.post_init() + +main(cfg,cfg_in) diff --git a/setup.cfg b/setup.cfg index 3f804e3..628ebe0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.2.dev10 + mmgen>=13.2.dev12 packages = mmgen_node_tools @@ -36,3 +36,4 @@ scripts = cmds/mmnode-halving-calculator cmds/mmnode-netrate cmds/mmnode-peerblocks + cmds/mmnode-ticker diff --git a/test/ref/ticker/ticker-btc.json b/test/ref/ticker/ticker-btc.json new file mode 100644 index 0000000..75c8909 --- /dev/null +++ b/test/ref/ticker/ticker-btc.json @@ -0,0 +1 @@ +{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23368.859557988893","price_btc":"1","volume_24h_usd":"24116251608.791744","market_cap_usd":"446560795287","circulating_supply":"19109225","total_supply":"19109231","max_supply":"21000000","percent_change_1h":"-0.23","percent_change_24h":"-1.87","percent_change_7d":"6.05","last_updated":"1659346445"} \ No newline at end of file diff --git a/test/ref/ticker/ticker-cfg.yaml b/test/ref/ticker/ticker-cfg.yaml new file mode 100644 index 0000000..504a7fb --- /dev/null +++ b/test/ref/ticker/ticker-cfg.yaml @@ -0,0 +1,24 @@ +proxy: http://asdfzxcv:32459 + +cachedir: + +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + coin2: + - ada-cardano + - algo-algorand + commodity: + - xau-gold-spot-token + - xag-silver-spot-token + - xbr-brent-crude-oil-spot + fiat: + - chf-swiss-franc-token + - eur-euro-token + index: + - dj30-dow-jones-30-token + - spx-sp-500 + - ndx-nasdaq-100-token diff --git a/test/ref/ticker/ticker-portfolio-bad.yaml b/test/ref/ticker/ticker-portfolio-bad.yaml new file mode 100644 index 0000000..0a2740e --- /dev/null +++ b/test/ref/ticker/ticker-portfolio-bad.yaml @@ -0,0 +1,6 @@ +btc-bitcoin: '1.23456789' +eth-ethereum: '2.345678901234567890' +xmr-monero: '4.567890123456' +ada-cardano: '123.45678901' +algo-algorand: '234.5678901' +noc-nocoin: '777.1234' diff --git a/test/ref/ticker/ticker-portfolio.yaml b/test/ref/ticker/ticker-portfolio.yaml new file mode 100644 index 0000000..332257f --- /dev/null +++ b/test/ref/ticker/ticker-portfolio.yaml @@ -0,0 +1,5 @@ +btc-bitcoin: '1.23456789' +eth-ethereum: '2.345678901234567890' +xmr-monero: '4.567890123456' +ada-cardano: '123.45678901' +algo-algorand: '234.5678901' diff --git a/test/ref/ticker/ticker.json b/test/ref/ticker/ticker.json new file mode 100644 index 0000000..2484e6d --- /dev/null +++ b/test/ref/ticker/ticker.json @@ -0,0 +1 @@ +[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"},{"id":"xau-gold-spot-token","name":"Gold Spot Token","symbol":"XAU","rank":"2634","price_usd":"1767.0700000000002","price_btc":"0.07608887785567188","volume_24h_usd":"11881071.852000002","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.33","percent_change_24h":"-0.19","percent_change_7d":"2.89","last_updated":"1659464759"},{"id":"xag-silver-spot-token","name":"Silver Spot Token","symbol":"XAG","rank":"4753","price_usd":"20.0265","price_btc":"0.0008623279849562342","volume_24h_usd":"445379.34674999997","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.61","percent_change_24h":"-1.54","percent_change_7d":"7.49","last_updated":"1659464759"},{"id":"xbr-brent-crude-oil-spot","name":"Brent Crude Oil Spot","symbol":"XBR","rank":"2616","price_usd":"100.39000000000001","price_btc":"0.0043227277062770015","volume_24h_usd":"4527788.105237802","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.89","percent_change_24h":"0.87","percent_change_7d":"1.02","last_updated":"1659464759"},{"id":"chf-swiss-franc-token","name":"Swiss Franc Token","symbol":"CHF","rank":"3234","price_usd":"1.0453383927173951","price_btc":"0.00004501158713651311","volume_24h_usd":"360978.80080814153","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.08","percent_change_24h":"-0.66","percent_change_7d":"0.65","last_updated":"1659464759"},{"id":"eur-euro-token","name":"Euro Token","symbol":"EUR","rank":"5116","price_usd":"1.0185784743417672","price_btc":"0.00004385932256255119","volume_24h_usd":"40587878.087926075","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"0.05","percent_change_24h":"-0.82","percent_change_7d":"0.6","last_updated":"1659464759"},{"id":"gbp-pound-sterling-token","name":"Pound Sterling Token","symbol":"GBP","rank":"3424","price_usd":"1.2181792934946465","price_btc":"0.00005245400321946659","volume_24h_usd":"5428622.322314565","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.61","percent_change_7d":"1.29","last_updated":"1659464759"},{"id":"dj30-dow-jones-30-token","name":"Dow Jones 30 Token","symbol":"DJ30","rank":"3192","price_usd":"32546","price_btc":"1.401409462381624","volume_24h_usd":"293564.92","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.1","percent_change_24h":"-0.91","percent_change_7d":"2.5","last_updated":"1659464759"},{"id":"spx-sp-500","name":"S&P 500 Token","symbol":"SPX","rank":"3342","price_usd":"4110.6","price_btc":"0.17699974608449287","volume_24h_usd":"780397.41","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"-0.27","percent_change_7d":"4.87","last_updated":"1659464759"},{"id":"ndx-nasdaq-100-token","name":"NASDAQ 100 Token","symbol":"NDX","rank":"5097","price_usd":"12948.199999999999","price_btc":"0.5575410188904857","volume_24h_usd":"11221816.494","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"0.08","percent_change_24h":"0.04","percent_change_7d":"7.21","last_updated":"1659464759"}] diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index fdc4800..f0c11d7 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -14,6 +14,7 @@ list_avail_tests() { echo " btc_rt - Bitcoin regtest" echo " bch_rt - Bitcoin Cash Node (BCH) regtest" echo " ltc_rt - Litecoin regtest" + echo " scripts - tests of scripts not requiring a coin daemon" echo " misc - miscellaneous tests that don't fit in the above categories" echo echo "AVAILABLE TEST GROUPS:" @@ -27,10 +28,10 @@ list_avail_tests() { } init_groups() { - dfl_tests='unit misc btc_rt bch_rt ltc_rt' + dfl_tests='unit misc scripts btc_rt bch_rt ltc_rt' extra_tests='' - noalt_tests='unit misc btc_rt' - quick_tests='unit misc btc_rt' + noalt_tests='unit misc scripts btc_rt' + quick_tests='unit misc scripts btc_rt' qskip_tests='bch_rt ltc_rt' } @@ -45,6 +46,11 @@ init_tests() { t_misc="- $test_py helpscreens" f_misc='Misc tests completed' + i_scripts='No-daemon scripts' + s_scripts="The following tests will test scripts not requiring a coin daemon" + t_scripts="- $test_py scripts" + f_scripts='No-daemon script tests completed' + i_btc_rt='Bitcoin regtest' s_btc_rt="The following tests will test various scripts using regtest mode" t_btc_rt="- $test_py regtest" diff --git a/test/test_py_d/cfg.py b/test/test_py_d/cfg.py index c56f74c..d2726c4 100755 --- a/test/test_py_d/cfg.py +++ b/test/test_py_d/cfg.py @@ -16,6 +16,7 @@ import os cmd_groups_dfl = { 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), + 'scripts': ('TestSuiteScripts',{'modname':'misc'}), 'regtest': ('TestSuiteRegtest',{}), } @@ -23,6 +24,7 @@ cmd_groups_extra = {} cfgs = { '1': {}, # regtest + '2': {}, # scripts } def fixup_cfgs(): pass diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 707f32f..c76bad6 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -12,6 +12,7 @@ ts_misc.py: Miscellaneous test groups for the test.py test suite """ +import shutil from ..include.common import * from .common import * from .ts_base import * @@ -48,3 +49,225 @@ class TestSuiteHelp(TestSuiteBase): def longhelpscreens(self): return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') + +refdir = os.path.join('test','ref','ticker') + +class TestSuiteScripts(TestSuiteBase): + 'scripts not requiring a coin daemon' + networks = ('btc',) + tmpdir_nums = [2] + passthru_opts = () + color = True + + cmd_group_in = ( + ('subgroup.ticker_setup', []), + ('subgroup.ticker', ['ticker_setup']), + ) + cmd_subgroups = { + 'ticker_setup': ( + "setup for 'ticker' subgroup", + ('ticker_setup', 'ticker setup'), + ), + 'ticker': ( + "'mmnode-ticker' script", + ('ticker1', 'ticker [--help)'), + ('ticker2', 'ticker (bad proxy)'), + ('ticker3', 'ticker [--cached-data]'), + ('ticker4', 'ticker [--cached-data --wide]'), + ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), + ('ticker6', 'ticker [--cached-data --wide --portfolio] (missing portfolio)'), + ('ticker7', 'ticker [--cached-data --wide --portfolio]'), + ('ticker8', 'ticker [--cached-data --wide --elapsed]'), + ('ticker9', 'ticker [--cached-data --wide --portfolio --elapsed --add-rows=fake-fakecoin:0.0123 --add-precision=2]'), + ('ticker10', 'ticker [--cached-data xmr:17.234]'), + ('ticker11', 'ticker [--cached-data xmr:17.234:btc]'), + ('ticker12', 'ticker [--cached-data --adjust=1.23 xmr:17.234:btc]'), + ('ticker13', 'ticker [--cached-data --wide --elapsed -c inr-indian-rupee:79.5 inr:200000:btc:0.1]'), + ('ticker14', 'ticker [--cached-data --wide --btc]'), + ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), + ) + } + + @property + def ticker_args(self): + return [ f'--cachedir={self.tmpdir}', '--proxy=http://asdfzxcv:32459' ] + + @property + def nt_datadir(self): + return os.path.join( g.data_dir_root, 'node_tools' ) + + def ticker_setup(self): + self.spawn('',msg_only=True) + shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) + shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) + return 'ok' + + def ticker(self,args=[],expect_list=None,cached=True): + t = self.spawn( + f'mmnode-ticker', + (['--cached-data'] if cached else []) + self.ticker_args + args ) + if expect_list: + t.match_expect_list(expect_list) + return t + + def ticker1(self): + t = self.ticker(['--help']) + t.expect('USAGE:') + return t + + def ticker2(self): + t = self.ticker(cached=False) + if not opt.skipping_deps: + t.expect('Creating') + t.expect('Creating') + t.expect('proxy host could not be resolved') + t.req_exit_val = 3 + return t + + def ticker3(self): + return self.ticker( + [], + [ + 'USD BTC', + 'BTC 23250.77 1.00000000 ETH 1659.66 0.07146397' + ]) + + + def ticker4(self): + return self.ticker( + ['--wide','--add-columns=eur,inr-indian-rupee:79.5'], + [ + r'EUR \(EURO TOKEN\) = 1.0186 USD ' + + r'INR \(INDIAN RUPEE\) = 0.012579 USD', + 'USD EUR INR BTC CHG_7d CHG_24h UPDATED', + 'BITCOIN', + r'ETHEREUM 1,659.66 1,629.3906 131,943.14 0.07146397 \+21.42 \+1.82', + r'MONERO 158.97 156.0734 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', + r'INDIAN RUPEE 0.01 0.0123 1.00 0.00000054 -- --', + ]) + + def ticker5(self): + shutil.copy2(os.path.join(refdir,'ticker-cfg.yaml'),self.nt_datadir) + t = self.ticker( + ['--wide','--adjust=-0.766'], + [ + 'Adjusting prices by -0.77%', + 'USD BTC CHG_7d CHG_24h UPDATED', + r'LITECOIN 58.56 0.00252162 \+12.79 \+0.40 2022-08-02 18:25:59', + r'MONERO 157.76 0.00679284 \+7.28 \+1.21' + ]) + os.unlink(os.path.join(self.nt_datadir,'ticker-cfg.yaml')) + return t + + def ticker6(self): + t = self.ticker( ['--wide','--portfolio'], None ) + t.expect('No portfolio') + t.req_exit_val = 1 + return t + + def ticker7(self): # demo + shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + t = self.ticker( + ['--wide','--portfolio'], + [ + 'USD BTC CHG_7d CHG_24h UPDATED', + r'ETHEREUM 1,659.66 0.07146397 \+21.42 \+1.82 2022-08-02 18:25:59', + 'CARDANO','ALGORAND', + 'PORTFOLIO','BITCOIN','ETHEREUM','MONERO','CARDANO','ALGORAND','TOTAL' + ]) + os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + return t + + def ticker8(self): + return self.ticker( + ['--wide','--elapsed'], + [ + 'USD BTC CHG_7d CHG_24h UPDATED', + r'BITCOIN 23,250.77 1.00000000 \+11.15 \+0.89 10 minutes ago' + ]) + + def ticker9(self): + shutil.copy2( + os.path.join(refdir,'ticker-portfolio-bad.yaml'), + os.path.join(self.nt_datadir,'ticker-portfolio.yaml') ) + t = self.ticker( + ['--wide','--portfolio','--elapsed','--add-rows=fake-fakecoin:0.0123','--add-precision=2'], + [ + 'USD BTC CHG_7d CHG_24h UPDATED', + r'BITCOIN 23,250.7741 1.0000000000 \+11.15 \+0.89 10 minutes ago', + r'FAKECOIN 81.3008 0.0034966927 -- -- just now', + r'\(no data for noc-nocoin\)', + ]) + os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + return t + + def ticker10(self): + return self.ticker( + ['XMR:17.234'], + [ + r'XMR \(MONERO\) = 158.97 USD ' + + 'Amount: 17.234 XMR', + 'SPOT PRICE', + 'BTC 0.11783441', + 'XMR 17.23400000', + 'XAU','NDX', + ]) + + def ticker11(self): + return self.ticker( + ['XMR:17.234:BTC'], + [ + r'XMR \(MONERO\) = 158.97 USD ' + + r'BTC \(BITCOIN\) = 23250.77 USD ' + + 'Amount: 17.234 XMR', + 'SPOT PRICE', + 'XMR 17.23400000 BTC 0.11783441', + ]) + + def ticker12(self): + return self.ticker( + ['--adjust=1.23','--wide','XMR:17.234:BTC'], + [ + r'XMR \(MONERO\) = 158.97 USD ' + + r'BTC \(BITCOIN\) = 23,250.77 USD ' + + 'Amount: 17.234 XMR', + r'Adjusting prices by \+1.23%', + 'SPOT PRICE ADJUSTED PRICE', + 'MONERO 17.23400000 17.44597820 2022-08-02 18:25:59 ' + + 'BITCOIN 0.11783441 0.11928377 2022-08-02 18:25:59', + ]) + + def ticker13(self): + return self.ticker( + ['-wE','-c','inr-indian-rupee:79.5','inr:200000:btc:0.1'], + [ + 'Offer: 200,000 INR', + 'Offered price differs from spot by -7.58%', + 'SPOT PRICE OFFERED PRICE UPDATED', + 'INDIAN RUPEE 200,000.00000000 184,843.65372424 10 minutes ago ' + + 'BITCOIN 0.10819955 0.10000000 10 minutes ago' + ]) + + def ticker14(self): + shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + t = self.ticker( + ['--btc','--wide','--portfolio','--elapsed'], + [ + 'PRICES', + r'BITCOIN 23,368.86 \+6.05 -1.87 33 hours, 2 minutes ago', + 'PORTFOLIO', + r'BITCOIN 28,850.44 \+6.05 -1.87 1.23456789' + ]) + os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + return t + + def ticker15(self): + return self.ticker( + ['--btc','--wide','--elapsed','-r','inr:79.5','btc:2:usd:45000'], + [ + r'BTC \(BITCOIN\) = 23,368.86 USD', + 'Offered price differs from spot by -3.72%', + 'SPOT PRICE OFFERED PRICE UPDATED', + 'BITCOIN 2.00000000 1.92563954 33 hours, 2 minutes ago ' + + 'US DOLLAR 46,737.71911598 45,000.00000000 33 hours, 2 minutes ago', + ]) From 015de30cdd4e5dcc2fad8c27dc36fd6a21e400f3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 5 Aug 2022 11:57:23 +0000 Subject: [PATCH 009/108] `mmnode-ticker`: support reverse USD rate in asset specifier Testing / demo: $ test/test.py -e scripts.ticker --- mmgen_node_tools/Ticker.py | 13 +++++++++---- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 20 ++++++++++++-------- test/test_py_d/ts_misc.py | 12 ++++++++++++ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index cafe815..618b701 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -295,13 +295,18 @@ def make_cfg(cmd_args,cfg_in): sym,id = (s.split('-')[0],s) if '-' in s else (s,None) return asset_tuple( sym.upper(), id.lower() if id else None ) - def parse_asset_triplet(s): + def parse_asset_triplet(s,reverse_ok=False): ss = s.split(':') - sym,amt = ( ss[0], Decimal(ss[1]) ) if len(ss) == 2 else ( s, None ) - return asset_triplet( *parse_asset_tuple(sym), amt ) + return asset_triplet( + *parse_asset_tuple(s if len(ss) == 1 else ss[0]), + ( + None if len(ss) == 1 else + 1 / Decimal(ss[1][:-1]) if reverse_ok and ss[1].lower().endswith('r') else + Decimal(ss[1]) + )) def parse_usr_asset_arg(s): - return tuple(parse_asset_triplet(ss) for ss in s.split(',')) if s else () + return tuple(parse_asset_triplet(ss,reverse_ok=True) for ss in s.split(',')) if s else () def parse_query_arg(s): ss = s.split(':') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 7fc0f94..fb75e93 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev4 +3.1.dev5 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 1949a66..d4a5533 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -76,11 +76,13 @@ ambiguous, the full ID must be used. Examples: ASSET SPECIFIERS consist of an ASSET followed by an optional colon and USD rate. If the asset is not in the source data (see --list-ids), the label -part of the ID may be arbitrarily chosen by the user. Examples: +part of the ID may be arbitrarily chosen by the user. When the letter ‘r’ +is appended to the USD rate, the rate is reversed, i.e. treated as ‘USD per +asset’ instead of ‘asset per USD’. Asset specifier examples: inr:79.5 - INR is not in the source data, so supply USD rate inr-indian-rupee:79.5 - same as above, but add an arbitrary label - ada-cardano:0.51 - ADA is in the source data, so use the listed ID + omr-omani-rial:2.59r - OMR is pegged to USD with fixed value of 2.59 USD A TRADE_SPECIFIER is a single argument in the format: @@ -121,9 +123,9 @@ configured cache directory, and run the script with the --cached-data option. To protect user privacy, all filtering and processing of data is performed client side so that the remote server does not know which assets are being examined. This means that data for ALL available assets (currently over 4000) -is fetched with each invocation of the script. A rate limit of {ratelimit} seconds -between calls is thus imposed to prevent abuse of the remote server. When the ---btc option is in effect, this limit is reduced to 10 seconds. To bypass the +is fetched with each invocation of the script. A rate limit of {L} seconds +between calls is thus imposed to prevent abuse of the remote server. When the +--btc option is in effect, this limit is reduced to {B} seconds. To bypass the rate limit entirely, use --cached-data. @@ -135,8 +137,9 @@ $ mmnode-ticker # Display BTC price only: $ mmnode-ticker --btc -# Wide display, add EUR and INR columns, INR rate, extra precision and proxy: -$ mmnode-ticker -w -c eur,inr-indian-rupee:79.5 -e2 -x http://vpnhost:8118 +# Wide display, add EUR and OMR columns, OMRUSD rate, extra precision and +# proxy: +$ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 # Wide display, use cached data from previous network query, show portfolio # (see above), pipe output to pager, add DOGE row: @@ -177,7 +180,8 @@ To add a portfolio, edit the file pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), api_host = api_host, api_url = api_url, - ratelimit = ratelimit, + L = ratelimit, + B = btc_ratelimit, ) } } diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index c76bad6..9815560 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -85,6 +85,7 @@ class TestSuiteScripts(TestSuiteBase): ('ticker13', 'ticker [--cached-data --wide --elapsed -c inr-indian-rupee:79.5 inr:200000:btc:0.1]'), ('ticker14', 'ticker [--cached-data --wide --btc]'), ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), + ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), ) } @@ -271,3 +272,14 @@ class TestSuiteScripts(TestSuiteBase): 'BITCOIN 2.00000000 1.92563954 33 hours, 2 minutes ago ' + 'US DOLLAR 46,737.71911598 45,000.00000000 33 hours, 2 minutes ago', ]) + + def ticker16(self): + return self.ticker( + ['--wide','--elapsed','-c','eur,omr-omani-rial:2.59r'], + [ + r'EUR \(EURO TOKEN\) = 1.0186 USD ' + + r'OMR \(OMANI RIAL\) = 2.5900 USD', + 'USD EUR OMR BTC CHG_7d CHG_24h UPDATED', + 'BITCOIN', + 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- just now' + ]) From 3fd9c971bf6b76b85e82fb44378c502b6707b254 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 6 Aug 2022 09:48:48 +0000 Subject: [PATCH 010/108] minor cleanups --- README.md | 23 ++++++--- mmgen_node_tools/Ticker.py | 87 +++++++++++++++++++-------------- mmgen_node_tools/main_ticker.py | 30 +++++++----- test/test_py_d/ts_misc.py | 2 +- 4 files changed, 86 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 088a344..9c72f4e 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ First, install [MMGen][6]. Then, - $ git clone https://github.com/mmgen/mmgen-node-tools - $ cd mmgen-node-tools - $ python3 -m build --no-isolation - $ python3 -m pip install --user dist/*.whl + $ git clone https://github.com/mmgen/mmgen-node-tools + $ cd mmgen-node-tools + $ python3 -m build --no-isolation + $ python3 -m pip install --user dist/*.whl Also make sure that `~/.local/bin` is in `PATH`. @@ -24,9 +24,18 @@ Also make sure that `~/.local/bin` is in `PATH`. *NOTE: the tests require that the MMGen and MMGen Node Tools repositories be located in the same directory.* - $ test/init.sh - $ test/test-release.sh -A # BTC-only testing - $ test/test-release.sh # Full testing +Initialize the test framework (must be run at least once after cloning, and +possibly again after a pull if tests have been updated): + + $ test/init.sh + +BTC-only testing: + + $ test/test-release.sh -A + +Full testing: + + $ test/test-release.sh - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 618b701..b667ccd 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -87,11 +87,11 @@ def gen_data(data): usr_wants = { 'id': ( {a.id for a in usr_assets if a.id} - - {a.id for a in usr_assets if a.amount and a.id} - {'usd-us-dollar'} ) + {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'} ) , 'symbol': ( {a.symbol for a in usr_assets if not a.id} - - {a.symbol for a in usr_assets if a.amount} - {'USD'} ), + {a.symbol for a in usr_assets if a.rate} - {'USD'} ), } found = { 'id': set(), 'symbol': set() } @@ -114,7 +114,7 @@ def gen_data(data): break for asset in (cfg.usr_rows + cfg.usr_columns): - if asset.amount: + if asset.rate: """ User-supplied rate overrides rate from source data. """ @@ -122,8 +122,8 @@ def gen_data(data): yield ( _id, { 'symbol': asset.symbol, 'id': _id, - 'price_usd': str(Decimal(1/asset.amount)), - 'price_btc': str(Decimal(1/asset.amount/btcusd)), + 'price_usd': str(Decimal(1/asset.rate)), + 'price_btc': str(Decimal(1/asset.rate/btcusd)), 'last_updated': int(now), }) @@ -280,47 +280,62 @@ def main(cfg_parm,cfg_in_parm): def make_cfg(cmd_args,cfg_in): def get_rows_from_cfg(add_data=None): - def create_row(e): - return asset_tuple(e.split('-')[0].upper(),e) def gen(): for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): yield(k) if add_data and k in add_data: v += tuple(add_data[k]) for e in v: - yield(create_row(e)) + yield parse_asset_id(e,True) return tuple(gen()) - def parse_asset_tuple(s): - sym,id = (s.split('-')[0],s) if '-' in s else (s,None) - return asset_tuple( sym.upper(), id.lower() if id else None ) - - def parse_asset_triplet(s,reverse_ok=False): - ss = s.split(':') - return asset_triplet( - *parse_asset_tuple(s if len(ss) == 1 else ss[0]), - ( - None if len(ss) == 1 else - 1 / Decimal(ss[1][:-1]) if reverse_ok and ss[1].lower().endswith('r') else - Decimal(ss[1]) - )) + def parse_asset_id(s,require_label=False): + sym,label = (*s.split('-',1),None)[:2] + if require_label and not label: + die(1,f'{s!r}: asset label is missing') + return asset_tuple( sym.upper(), (s.lower() if label else None) ) def parse_usr_asset_arg(s): - return tuple(parse_asset_triplet(ss,reverse_ok=True) for ss in s.split(',')) if s else () + """ + asset_id[:rate] + """ + def parse_parm(s): + ss = s.split(':') + assert len(ss) in (1,2), f'{s}: malformed argument' + asset_id,rate = (*ss,None)[:2] + parsed_id = parse_asset_id(asset_id) + + return asset_data( + symbol = parsed_id.symbol, + id = parsed_id.id, + amount = None, + rate = ( + None if rate is None else + 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else + Decimal(rate) )) + + return tuple(parse_parm(s2) for s2 in s.split(',')) if s else () def parse_query_arg(s): + """ + asset_id:amount[:to_asset_id[:to_amount]] + """ + def parse_query_asset(asset_id,amount): + parsed_id = parse_asset_id(asset_id) + return asset_data( + symbol = parsed_id.symbol, + id = parsed_id.id, + amount = None if amount is None else Decimal(amount), + rate = None ) + ss = s.split(':') - if len(ss) == 2: - return query_tuple( - asset = parse_asset_triplet(s), - to_asset = None ) - elif len(ss) in (3,4): - return query_tuple( - asset = parse_asset_triplet(':'.join(ss[:2])), - to_asset = parse_asset_triplet(':'.join(ss[2:])), - ) - else: - die(1,f'{s}: malformed argument') + assert len(ss) in (2,3,4), f'{s}: malformed argument' + asset_id,amount,to_asset_id,to_amount = (*ss,None,None)[:4] + + return query_tuple( + asset = parse_query_asset(asset_id,amount), + to_asset = parse_query_asset(to_asset_id,to_amount) if to_asset_id else None + ) def gen_uniq(obj_list,key,preload=None): found = set([getattr(obj,key) for obj in preload if hasattr(obj,key)] if preload else ()) @@ -339,7 +354,7 @@ def make_cfg(cmd_args,cfg_in): def get_portfolio_assets(ret=()): if cfg_in.portfolio and opt.portfolio: - ret = tuple( asset_tuple(e.split('-')[0].upper(),e) for e in cfg_in.portfolio ) + ret = (parse_asset_id(e,True) for e in cfg_in.portfolio) return ( 'portfolio', tuple(e for e in ret if (not opt.btc) or e.symbol == 'BTC') ) def get_portfolio(): @@ -357,7 +372,7 @@ def make_cfg(cmd_args,cfg_in): def create_rows(): rows = ( ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin',parse_asset_tuple('btc-bitcoin')) if opt.btc else + ('bitcoin',parse_asset_id('btc-bitcoin')) if opt.btc else get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if opt.add_columns else None ) ) @@ -385,7 +400,7 @@ def make_cfg(cmd_args,cfg_in): 'portfolio' ]) query_tuple = namedtuple('query',['asset','to_asset']) - asset_triplet = namedtuple('asset_triplet',['symbol','id','amount']) + asset_data = namedtuple('asset_data',['symbol','id','amount','rate']) asset_tuple = namedtuple('asset_tuple',['symbol','id']) usr_rows = parse_usr_asset_arg(opt.add_rows) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index d4a5533..e843475 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -67,22 +67,28 @@ user’s portfolio, while trading mode displays the price of a given quantity of an asset in relation to other assets, optionally comparing an offered price to the spot price. -ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID consisting of -symbol plus label (e.g. ‘xmr-monero’). In cases where the symbol is -ambiguous, the full ID must be used. Examples: +ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids) +consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the +symbol is ambiguous, the full ID must be used. Examples: chf - specify asset by symbol chf-swiss-franc-token - same as above, but use full ID instead of symbol -ASSET SPECIFIERS consist of an ASSET followed by an optional colon and USD -rate. If the asset is not in the source data (see --list-ids), the label -part of the ID may be arbitrarily chosen by the user. When the letter ‘r’ -is appended to the USD rate, the rate is reversed, i.e. treated as ‘USD per -asset’ instead of ‘asset per USD’. Asset specifier examples: +ASSET SPECIFIERS have the following format: - inr:79.5 - INR is not in the source data, so supply USD rate - inr-indian-rupee:79.5 - same as above, but add an arbitrary label - omr-omani-rial:2.59r - OMR is pegged to USD with fixed value of 2.59 USD + ASSET[:RATE] + +If the asset referred to by ASSET is not in the source data (see --list-ids), +an arbitrarily chosen label may be used. RATE is the USD exchange rate of +the asset. When RATE is postfixed with the letter ‘r’, its meaning is +reversed, i.e. interpreted as ‘ASSET/USD’ instead of ‘USD/ASSET’. Asset +specifier examples: + + inr:79.5 - INR is not in the source data, so supply rate of + 79.5 INR to the Dollar (USD/INR) + inr:0.01257r - same as above, but use reverse rate (INR/USD) + inr-indian-rupee:79.5 - same as first example, but add an arbitrary label + omr-omani-rial:2.59r - Omani Rial is pegged to the Dollar at 2.59 USD A TRADE_SPECIFIER is a single argument in the format: @@ -137,7 +143,7 @@ $ mmnode-ticker # Display BTC price only: $ mmnode-ticker --btc -# Wide display, add EUR and OMR columns, OMRUSD rate, extra precision and +# Wide display, add EUR and OMR columns, OMR/USD rate, extra precision and # proxy: $ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 9815560..7295da3 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -280,6 +280,6 @@ class TestSuiteScripts(TestSuiteBase): r'EUR \(EURO TOKEN\) = 1.0186 USD ' + r'OMR \(OMANI RIAL\) = 2.5900 USD', 'USD EUR OMR BTC CHG_7d CHG_24h UPDATED', - 'BITCOIN', + r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- just now' ]) From f9ba7b89c8bedb401b07812b856873ec60838bc5 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 6 Aug 2022 09:48:48 +0000 Subject: [PATCH 011/108] `mmnode-ticker`: support rate asset in asset specifier This feature adds convenience for users whose native currencies are pegged to assets other than the US Dollar. Example: # Add a column for Bulgarian Lev, which is pegged to the Euro at 0.5113 EUR: $ mmnode-ticker -wE -c eur,bgn-bulgarian-lev:0.5113r:eur Testing / demo: $ test/test.py -e scripts.ticker --- mmgen_node_tools/Ticker.py | 31 +++++++++++++++++++++---------- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 14 +++++++++----- test/test_py_d/ts_misc.py | 23 +++++++++++++++++++++++ 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index b667ccd..14163eb 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -83,18 +83,24 @@ def gen_data(data): 'id': {r.id for r in cfg.rows if getattr(r,'id',None)} - {'usd-us-dollar'}, 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, } + usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) + usr_rate_assets_want = { + 'id': {a.id for a in usr_rate_assets if a.id}, + 'symbol': {a.symbol for a in usr_rate_assets if not a.id} + } usr_assets = cfg.usr_rows + cfg.usr_columns + tuple(c for c in (cfg.query or ()) if c) usr_wants = { 'id': ( - {a.id for a in usr_assets if a.id} - + {a.id for a in usr_assets + usr_rate_assets if a.id} - {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'} ) , 'symbol': ( - {a.symbol for a in usr_assets if not a.id} - + {a.symbol for a in usr_assets + usr_rate_assets if not a.id} - {a.symbol for a in usr_assets if a.rate} - {'USD'} ), } found = { 'id': set(), 'symbol': set() } + rate_assets = {} for k in ['id','symbol']: wants = rows_want[k] | usr_wants[k] @@ -105,6 +111,8 @@ def gen_data(data): die(1,dup_sym_errmsg(d[k])) yield (d['id'],d) found[k].add(d[k]) + if d[k] in usr_rate_assets_want[k]: + rate_assets[d['symbol']] = d # NB: using symbol instead of ID if k == 'id' and len(found[k]) == len(wants): break @@ -119,11 +127,12 @@ def gen_data(data): User-supplied rate overrides rate from source data. """ _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() + ra_rate = Decimal(rate_assets[asset.rate_asset.symbol]['price_usd']) if asset.rate_asset else 1 yield ( _id, { 'symbol': asset.symbol, 'id': _id, - 'price_usd': str(Decimal(1/asset.rate)), - 'price_btc': str(Decimal(1/asset.rate/btcusd)), + 'price_usd': str(Decimal(ra_rate/asset.rate)), + 'price_btc': str(Decimal(ra_rate/asset.rate/btcusd)), 'last_updated': int(now), }) @@ -297,12 +306,12 @@ def make_cfg(cmd_args,cfg_in): def parse_usr_asset_arg(s): """ - asset_id[:rate] + asset_id[:rate[:rate_asset]] """ def parse_parm(s): ss = s.split(':') - assert len(ss) in (1,2), f'{s}: malformed argument' - asset_id,rate = (*ss,None)[:2] + assert len(ss) in (1,2,3), f'{s}: malformed argument' + asset_id,rate,rate_asset = (*ss,None,None)[:3] parsed_id = parse_asset_id(asset_id) return asset_data( @@ -312,7 +321,8 @@ def make_cfg(cmd_args,cfg_in): rate = ( None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else - Decimal(rate) )) + Decimal(rate) ), + rate_asset = parse_asset_id(rate_asset) if rate_asset else None ) return tuple(parse_parm(s2) for s2 in s.split(',')) if s else () @@ -326,7 +336,8 @@ def make_cfg(cmd_args,cfg_in): symbol = parsed_id.symbol, id = parsed_id.id, amount = None if amount is None else Decimal(amount), - rate = None ) + rate = None, + rate_asset = None ) ss = s.split(':') assert len(ss) in (2,3,4), f'{s}: malformed argument' @@ -400,7 +411,7 @@ def make_cfg(cmd_args,cfg_in): 'portfolio' ]) query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate']) + asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset']) asset_tuple = namedtuple('asset_tuple',['symbol','id']) usr_rows = parse_usr_asset_arg(opt.add_rows) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index fb75e93..2590359 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev5 +3.1.dev6 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index e843475..9a32c04 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -76,19 +76,20 @@ symbol is ambiguous, the full ID must be used. Examples: ASSET SPECIFIERS have the following format: - ASSET[:RATE] + ASSET[:RATE[:RATE_ASSET]] If the asset referred to by ASSET is not in the source data (see --list-ids), -an arbitrarily chosen label may be used. RATE is the USD exchange rate of -the asset. When RATE is postfixed with the letter ‘r’, its meaning is -reversed, i.e. interpreted as ‘ASSET/USD’ instead of ‘USD/ASSET’. Asset -specifier examples: +an arbitrarily chosen label may be used. RATE is the exchange rate of the +asset in relation to RATE_ASSET, if present, otherwise USD. When RATE is +postfixed with the letter ‘r’, its meaning is reversed, i.e. interpreted as +‘ASSET/RATE_ASSET’ instead of ‘RATE_ASSET/ASSET’. Asset specifier examples: inr:79.5 - INR is not in the source data, so supply rate of 79.5 INR to the Dollar (USD/INR) inr:0.01257r - same as above, but use reverse rate (INR/USD) inr-indian-rupee:79.5 - same as first example, but add an arbitrary label omr-omani-rial:2.59r - Omani Rial is pegged to the Dollar at 2.59 USD + bgn-bg-lev:0.5113r:eur - Bulgarian Lev is pegged to the Euro at 0.5113 EUR A TRADE_SPECIFIER is a single argument in the format: @@ -147,6 +148,9 @@ $ mmnode-ticker --btc # proxy: $ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 +# Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate: +$ mmnode-ticker -wE -c eur,bgn-bulgarian-lev:0.5113r:eur + # Wide display, use cached data from previous network query, show portfolio # (see above), pipe output to pager, add DOGE row: $ mmnode-ticker -wCFP -r doge diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 7295da3..0d39c7e 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -86,6 +86,8 @@ class TestSuiteScripts(TestSuiteBase): ('ticker14', 'ticker [--cached-data --wide --btc]'), ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), + ('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'), + ('ticker18', 'ticker [--cached-data --wide --elapsed -c eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'), ) } @@ -283,3 +285,24 @@ class TestSuiteScripts(TestSuiteBase): r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- just now' ]) + + def ticker17(self): + # BGN pegged at 0.5113 EUR + return self.ticker( + ['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eur'], + [ + r'BGN \(BULGARIAN LEV\) = 0.52080 USD', + 'USD BGN BTC CHG_7d CHG_24h UPDATED', + 'BITCOIN 23,250.77 44,644.414 1.00000000', + 'BULGARIAN LEV 0.52 1.000 0.00002240', + ]) + + def ticker18(self): + return self.ticker( + ['--wide','--elapsed','-c','eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'], + [ + r'BGN \(BULGARIAN LEV\) = 0.52080 USD', + 'USD EUR BGN BTC CHG_7d CHG_24h UPDATED', + 'BITCOIN 23,250.77 22,826.6890 44,644.414 1.00000000', + 'BULGARIAN LEV 0.52 0.5113 1.000 0.00002240', + ]) From cb93e5275544b80469844018c73a2bb6bea443f5 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 17 Oct 2022 18:35:55 +0000 Subject: [PATCH 012/108] update for MMGen v13.3.dev6 --- README.md | 2 +- mmgen_node_tools/Ticker.py | 12 +++++++----- mmgen_node_tools/Util.py | 13 ------------- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- mmgen_node_tools/main_blocks_info.py | 2 +- mmgen_node_tools/main_feeview.py | 9 ++++----- mmgen_node_tools/main_halving_calculator.py | 2 +- mmgen_node_tools/main_netrate.py | 2 +- mmgen_node_tools/main_peerblocks.py | 2 +- mmgen_node_tools/main_ticker.py | 1 + mmgen_node_tools/main_txfind.py | 2 +- setup.cfg | 2 +- test/test-release.d/cfg.sh | 10 ++++++++++ test/test_py_d/ts_misc.py | 6 +++--- 15 files changed, 36 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 9c72f4e..673155e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MMGen Node Tools -Helper utilities for Bitcoin and forkcoin full nodes. +### Terminal-based utilities for Bitcoin and forkcoin full nodes Requires modules from the [MMGen online/offline cryptocurrency wallet][6]. diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 14163eb..7fe8b1d 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -31,7 +31,7 @@ from collections import namedtuple from mmgen.opts import opt from mmgen.globalvars import g from mmgen.color import * -from mmgen.util import die,fmt_list,msg,msg_r,Msg,do_pager,suf,fmt +from mmgen.util import die,fmt_list,msg,msg_r,Msg,vmsg,suf,fmt,stdout_or_pager homedir = os.getenv('HOME') cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') @@ -189,6 +189,7 @@ def get_src_data(curl_cmd): elapsed = int(time.time() - os.stat(fn).st_mtime) if elapsed >= timeout: msg_r(f'Fetching data from {api_host}...') + vmsg('') try: cp = run(curl_cmd,check=True,stdout=PIPE) except CalledProcessError as e: @@ -244,10 +245,10 @@ def main(cfg_parm,cfg_in_parm): 'curl', '--tr-encoding', '--compressed', # adds 'Accept-Encoding: gzip' - '--silent', '--header', 'Accept: application/json', ] + (['--proxy', cfg.proxy] if cfg.proxy else []) + + (['--silent'] if not opt.verbose else []) + [api_url + ('/btc-bitcoin' if cfg.btc_only else '')] ) @@ -276,14 +277,15 @@ def main(cfg_parm,cfg_in_parm): parsed_json = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) if opt.list_ids: + from mmgen.ui import do_pager do_pager('\n'.join(e['id'] for e in parsed_json)) return global now now = 1659465400 if g.test_suite else time.time() # 1659524400 1659445900 - (do_pager if opt.pager else Msg)( - '\n'.join(getattr(Ticker,cfg.clsname)(dict(gen_data(parsed_json))).gen_output()) + stdout_or_pager( + '\n'.join(getattr(Ticker,cfg.clsname)(dict(gen_data(parsed_json))).gen_output()) + '\n' ) def make_cfg(cmd_args,cfg_in): @@ -481,7 +483,7 @@ class Ticker: def format_last_update_col(self,cross_assets=()): if opt.elapsed: - from .Util import format_elapsed_hr + from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) # ticker API diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 7fdc409..9060f68 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -33,19 +33,6 @@ def get_day_hms(t=None,utc=False): ret = (time.localtime,time.gmtime)[utc](secs) return '{:04}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*ret[0:6]) -def format_elapsed_hr(t,now=None,cached={}): - now = now or time.time() - e = int(now - t) - if not e in cached: - h = abs(e) // 3600 - m = abs(e) // 60 % 60 - cached[e] = '{a}{b} minute{c} {d}'.format( - a = '{} hour{}, '.format(h,suf(h)) if h else '', - b = m, - c = suf(m), - d = 'ago' if e > 0 else 'in the future' ) if (h or m) else 'just now' - return cached[e] - def do_system(cmd,testing=False,shell=False): if testing: from mmgen.util import msg diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 2590359..58dfba3 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev6 +3.1.dev7 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index c638ed6..4218f6a 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -121,7 +121,7 @@ async def main(req_addrs): height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') - from mmgen.base_proto.bitcoin.misc import scantxoutset + from mmgen.proto.btc.misc import scantxoutset res = await scantxoutset( rpc, [f'addr({addr})' for addr in addrs] ) if not res['success']: @@ -138,7 +138,7 @@ async def main(req_addrs): addr = re.match('addr\((.*?)\)',unspent['desc'])[1] addr_data[addr].append(unspent) else: - from mmgen.base_proto.bitcoin.tx.base import scriptPubKey2addr + from mmgen.proto.btc.tx.base import scriptPubKey2addr for unspent in sorted(res['unspents'],key=lambda x: x['height']): addr = scriptPubKey2addr( proto, unspent['scriptPubKey'] )[0] addr_data[addr].append(unspent) @@ -165,6 +165,6 @@ if len(cmd_args) < 1: from mmgen.obj import CoinTxID,Int try: - run_session(main(cmd_args)) + async_run(main(cmd_args)) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index a7dff8d..117da98 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -176,4 +176,4 @@ async def main(): m.finalize_output() -run_session(main()) +async_run(main()) diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 276c870..e5f2f00 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -21,6 +21,7 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool """ from mmgen.common import * +from mmgen.util2 import int2bytespec,parse_bytespec min_prec,max_prec,dfl_prec = (0,6,4) fee_brackets = [ @@ -213,10 +214,8 @@ async def main(): log(mempool,'mempool.json') data = create_data(proto.coin_amt,mempool) - - (do_pager if opt.pager else print)( + stdout_or_pager( '\n'.join(gen_header(c.host,await c.call('getblockcount'))) + '\n' + - '\n'.join(gen_body(data)) - ) + '\n'.join(gen_body(data)) + '\n' ) -run_session(main()) +async_run(main()) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 423df0e..c804d8f 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -179,4 +179,4 @@ async def main(): else: print_current_stats() -run_session(main()) +async_run(main()) diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index b1fb919..fd86983 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -68,6 +68,6 @@ async def main(): rs,ss,ts = r,s,t try: - run_session(main()) + async_run(main()) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 87df582..b775b29 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -171,7 +171,7 @@ ERASE_ALL,ERASE_LINE,CUR_HOME,CUR_HIDE,CUR_SHOW = ( '\033[J','\033[K','\033[H','\033[?25l','\033[?25h') try: - run_session(main()) + async_run(main()) except: from subprocess import run run(['stty','sane']) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 9a32c04..a25e884 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -52,6 +52,7 @@ opts_data = { -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column -U, --print-curl Print cURL command to standard output and exit +-v, --verbose Be more verbose -w, --wide Display all optional columns (equivalent to -punT) -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to disable. Consult the curl manpage for --proxy usage. diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 3c7935d..2672a11 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -91,4 +91,4 @@ msgs = msg_data['quiet' if opt.quiet else 'normal'] if len(cmd_args) != 1: die(1,'One transaction ID must be specified') -sys.exit(run_session(main(cmd_args[0]))) +sys.exit(async_run(main(cmd_args[0]))) diff --git a/setup.cfg b/setup.cfg index 628ebe0..f3d9a66 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.2.dev12 + mmgen>=13.3.dev6 packages = mmgen_node_tools diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index f0c11d7..53dd32a 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -8,6 +8,16 @@ # https://github.com/mmgen/mmgen-node-tools # https://gitlab.com/mmgen/mmgen-node-tools +# Testing status +# mmnode-addrbal OK +# mmnode-blocks-info - +# mmnode-feeview - +# mmnode-halving-calculator OK +# mmnode-netrate - +# mmnode-peerblocks - +# mmnode-ticker OK +# mmnode-txfind - + list_avail_tests() { echo "AVAILABLE TESTS:" echo " unit - unit tests" diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 0d39c7e..7801f64 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -257,7 +257,7 @@ class TestSuiteScripts(TestSuiteBase): ['--btc','--wide','--portfolio','--elapsed'], [ 'PRICES', - r'BITCOIN 23,368.86 \+6.05 -1.87 33 hours, 2 minutes ago', + r'BITCOIN 23,368.86 \+6.05 -1.87 1 day 9 hours 2 minutes ago', 'PORTFOLIO', r'BITCOIN 28,850.44 \+6.05 -1.87 1.23456789' ]) @@ -271,8 +271,8 @@ class TestSuiteScripts(TestSuiteBase): r'BTC \(BITCOIN\) = 23,368.86 USD', 'Offered price differs from spot by -3.72%', 'SPOT PRICE OFFERED PRICE UPDATED', - 'BITCOIN 2.00000000 1.92563954 33 hours, 2 minutes ago ' + - 'US DOLLAR 46,737.71911598 45,000.00000000 33 hours, 2 minutes ago', + 'BITCOIN 2.00000000 1.92563954 1 day 9 hours 2 minutes ago ' + + 'US DOLLAR 46,737.71911598 45,000.00000000 1 day 9 hours 2 minutes ago', ]) def ticker16(self): From 8ca7f43ae87c63678b0c6acdafc0c67c75f7c013 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 21 Oct 2022 10:39:02 +0000 Subject: [PATCH 013/108] mmnode-blocks-info: support BCH; add test --- mmgen_node_tools/BlocksInfo.py | 89 +++++++++++++++++----------- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_blocks_info.py | 7 ++- setup.cfg | 2 +- test/test-release.d/cfg.sh | 2 +- test/test_py_d/ts_regtest.py | 50 ++++++++++++++-- test/unit_tests_d/ut_BlocksInfo.py | 5 ++ 7 files changed, 113 insertions(+), 44 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 28395b4..27dfa08 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -23,7 +23,6 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of block import re,json from collections import namedtuple from time import strftime,gmtime -from decimal import Decimal from mmgen.common import * from mmgen.rpc import json_encoder @@ -46,14 +45,14 @@ class BlocksInfo: 'totalfee': bf( 'tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None ), 'size': bf( None, 'bs', '{:>7}', '', 'Size', 'total_size', None ), 'weight': bf( None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None ), - 'fee90': bf( None, 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4 ), - 'fee75': bf( None, 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3 ), - 'fee50': bf( None, 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2 ), - 'fee25': bf( None, 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1 ), - 'fee10': bf( None, 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0 ), - 'fee_max': bf( None, 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None ), - 'fee_avg': bf( None, 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None ), - 'fee_min': bf( None, 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None ), + 'fee90': bf( 'fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4 ), + 'fee75': bf( 'fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3 ), + 'fee50': bf( 'fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2 ), + 'fee25': bf( 'fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1 ), + 'fee10': bf( 'fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0 ), + 'fee_max': bf( 'fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None ), + 'fee_avg': bf( 'fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None ), + 'fee_min': bf( 'fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None ), 'nTx': bf( None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None ), 'inputs': bf( None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None ), 'outputs': bf( None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None ), @@ -101,24 +100,6 @@ class BlocksInfo: noindent_stats = ['col_avg'] avg_stats_skip = {'block', 'hash', 'date', 'version','miner'} - stats_deps = { - 'avg': set(fields) - avg_stats_skip, - 'col_avg':set(fields) - avg_stats_skip, - 'mini_avg':{'interval','size','weight'}, - 'total': {'interval','subsidy','totalfee','nTx','inputs','outputs','utxo_inc'}, - 'range': {}, - 'diff': {}, - } - - fmt_funcs = { - 'da': lambda arg: strftime('%Y-%m-%d %X',gmtime(arg)), - 'td': lambda arg: ( - '-{:02}:{:02}'.format(abs(arg)//60,abs(arg)%60) if arg < 0 else - ' {:02}:{:02}'.format(arg//60,arg%60) ), - 'tf': lambda arg: '{:.8f}'.format(arg * Decimal('0.00000001')), - 'su': lambda arg: str(arg * Decimal('0.00000001')).rstrip('0').rstrip('.'), - 'di': lambda arg: '{:.2e}'.format(arg), - } range_data = namedtuple('parsed_range_data',['first','last','from_tip','nblocks','step']) @@ -186,8 +167,44 @@ class BlocksInfo: self.opt = opt self.tip = rpc.blockcount + from_satoshi = self.rpc.proto.coin_amt.satoshi + to_satoshi = 1 / from_satoshi + self.block_list,self.first,self.last,self.step = parse_cmd_args() + have_segwit = self.rpc.info('segwit_is_active') + + if not have_segwit: + del self.fields['weight'] + self.dfl_fields = tuple(f for f in self.dfl_fields if f != 'weight') + + self.stats_deps = { + 'avg': set(self.fields) - self.avg_stats_skip, + 'col_avg': set(self.fields) - self.avg_stats_skip, + 'mini_avg': {'interval','size'} | ({'weight'} if have_segwit else set()), + 'total': {'interval','subsidy','totalfee','nTx','inputs','outputs','utxo_inc'}, + 'range': {}, + 'diff': {}, + } + + self.fmt_funcs = { + 'da': lambda arg: strftime('%Y-%m-%d %X',gmtime(arg)), + 'td': lambda arg: ( + '-{:02}:{:02}'.format(abs(arg)//60,abs(arg)%60) if arg < 0 else + ' {:02}:{:02}'.format(arg//60,arg%60) ), + 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), + 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), + 'fe': lambda arg: str(arg), + 'di': lambda arg: '{:.2e}'.format(arg), + } + + if g.coin == 'BCH': + self.fmt_funcs.update({ + 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), + 'fe': lambda arg: str(int(arg * to_satoshi)), + 'tf': lambda arg: '{:.8f}'.format(arg), + }) + self.fnames = tuple( [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] if opt.header_info else get_fields() if opt.fields else @@ -222,7 +239,8 @@ class BlocksInfo: self.bs_keys = set( [v.key1 for v in self.fvals if v.src == 'bs'] + - ['total_size','total_weight'] + ['total_size'] + + (['total_weight'] if have_segwit else []) ) if 'miner' in self.fnames: @@ -406,7 +424,8 @@ class BlocksInfo: await self.rpc.call('getblockstats',hdr['hash'],list(self.bs_keys)) ) self.total_bytes += bs['total_size'] - self.total_weight += bs['total_weight'] + if 'total_weight' in bs: + self.total_weight += bs['total_weight'] blk_data['bs'] = bs if 'miner' in self.fnames: @@ -506,7 +525,7 @@ class BlocksInfo: async def create_diff_stats(self): c = self.rpc - rel = self.tip % 2016 + rel = self.tip % self.rpc.proto.diff_adjust_interval tip_hdr = ( self.hdrs[-1] if self.hdrs[-1]['height'] == self.tip else @@ -520,14 +539,14 @@ class BlocksInfo: sample_blks = rel bdi = ( tip_hdr['time'] - rel_hdr['time'] ) / rel else: - sample_blks = min_sample_blks + sample_blks = min(min_sample_blks,self.tip) start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) diff_adj = float(tip_hdr['difficulty'] / start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] bdi = ((time1 * diff_adj) + time2) / sample_blks - rem = 2016 - rel + rem = self.rpc.proto.diff_adjust_interval - rel return ( 'difficulty', ( 'Difficulty Statistics:', @@ -572,13 +591,15 @@ class BlocksInfo: spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'mb_per_hour': spec_conv(0, '{:.4f}'), + 'mb_per_hour': spec_conv(0, '{}'), }, spec_vals = ( spec_val( 'mb_per_hour', 'MB/hr', 'interval', lambda values: 'bs' in self.deps, - lambda values: (self.total_bytes / 10000) / (self.total_solve_time / 36) + lambda values: ( + '{:.4f}'.format((self.total_bytes / 10000) / (self.total_solve_time / 36)) + if self.total_solve_time else 'N/A' ), ), ) ) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 58dfba3..2d22ff0 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev7 +3.1.dev8 diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 117da98..9295cd0 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -148,6 +148,11 @@ This program requires a txindex-enabled daemon for correct operation. f = fmt_list(BlocksInfo.fields,fmt='bare'), s = fmt_list(BlocksInfo.all_stats,fmt='bare'), p = g.prog_name ) + }, + 'code': { + 'notes': lambda proto,s: s.format( + adj_interval = proto.diff_adjust_interval, + ) } } @@ -158,7 +163,7 @@ async def main(): from mmgen.protocol import init_proto_from_opts from mmgen.rpc import rpc_init - proto = init_proto_from_opts() + proto = init_proto_from_opts(need_amt=True) cls = JSONBlocksInfo if opt.json else BlocksInfo diff --git a/setup.cfg b/setup.cfg index f3d9a66..327cad7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev6 + mmgen>=13.3.dev8 packages = mmgen_node_tools diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 53dd32a..59db1dd 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -10,7 +10,7 @@ # Testing status # mmnode-addrbal OK -# mmnode-blocks-info - +# mmnode-blocks-info OK # mmnode-feeview - # mmnode-halving-calculator OK # mmnode-netrate - diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 4fb3eec..d6b037e 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -45,6 +45,7 @@ class TestSuiteRegtest(TestSuiteBase): ('subgroup.halving_calculator', []), ('subgroup.fund_addrbal', []), ('subgroup.addrbal', ['fund_addrbal']), + ('subgroup.blocks_info', ['addrbal']), ('stop', 'stopping regtest daemon'), ) cmd_subgroups = { @@ -74,7 +75,14 @@ class TestSuiteRegtest(TestSuiteBase): ('addrbal_nobal3', 'getting address balances (one null balance)'), ('addrbal_nobal3_tabular1', 'getting address balances (one null balance, tabular output)'), ('addrbal_nobal3_tabular2', 'getting address balances (one null balance, tabular, show first block)'), - ) + ), + 'blocks_info': ( + "'mmnode-blocks-info' script", + ('blocks_info1', "blocks-info (--help)"), + ('blocks_info2', "blocks-info (no args)"), + ('blocks_info3', "blocks-info +100"), + ('blocks_info4', "blocks-info --miner-info --fields=all --stats=all +1"), + ), } def __init__(self,trunner,cfgs,spawn): @@ -86,11 +94,6 @@ class TestSuiteRegtest(TestSuiteBase): self.proto = init_proto(self.proto.coin,network='regtest',need_amt=True) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) - os.environ['MMGEN_TEST_SUITE_REGTEST'] = '1' - - def __del__(self): - os.environ['MMGEN_TEST_SUITE_REGTEST'] = '' - def setup(self): stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) from shutil import rmtree @@ -207,6 +210,41 @@ class TestSuiteRegtest(TestSuiteBase): self.addrs[3] + ' - - - -', ]) + def blocks_info(self,args,expect_list): + t = self.spawn('mmnode-blocks-info',args) + t.match_expect_list(expect_list) + return t + + def blocks_info1(self): + return self.blocks_info( args1 + ['--help'], ['USAGE:','OPTIONS:']) + + def blocks_info2(self): + return self.blocks_info( args1, [ + 'Current height: 396', + ]) + + def blocks_info3(self): + return self.blocks_info( args1 + ['+100'], [ + 'Range: 297-396', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) + + def blocks_info4(self): + n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if g.coin == 'BCH' else (2,1,4,6,3,12) + return self.blocks_info( args1 + ['--miner-info','--fields=all','--stats=all','+3'], [ + 'Averages', + f'nTx: {n1}', + f'Inputs: {i1}', + f'Outputs: {o1}', + 'Totals', + f'nTx: {n2}', + f'Inputs: {i2}', + f'Outputs: {o2}', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) + def stop(self): if opt.no_daemon_stop: self.spawn('',msg_only=True) diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/unit_tests_d/ut_BlocksInfo.py index 1448e97..e83cca0 100755 --- a/test/unit_tests_d/ut_BlocksInfo.py +++ b/test/unit_tests_d/ut_BlocksInfo.py @@ -65,6 +65,11 @@ fields_vecs = ( class dummyRPC: blockcount = tip + def info(self,arg): + return True + class proto: + class coin_amt: + satoshi = 0.00000001 class dummyOpt: fields = None From 38a55b57f127197229de9f47d47104d7ff74cc87 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 24 Oct 2022 17:38:11 +0000 Subject: [PATCH 014/108] minor fixes --- mmgen_node_tools/main_blocks_info.py | 19 +++++++++---------- test/test_py_d/ts_misc.py | 4 ++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 9295cd0..eb12a5c 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -74,13 +74,13 @@ block number is expected. If the requested range ends at the current chain tip, an estimate of the next difficulty adjustment is also displayed. The estimate is based on the average -Block Discovery Interval from the beginning of the current 2016-block period. +Block Discovery Interval from the beginning of the current {I}-block period. All fee fields except for 'totalfee' are in satoshis per virtual byte. -AVAILABLE FIELDS: {f} +AVAILABLE FIELDS: {F} -AVAILABLE STATS: {s} +AVAILABLE STATS: {S} EXAMPLES: @@ -110,7 +110,7 @@ EXAMPLES: $ {p} -o block,date,hash 245798 170 624044 Display every difficulty adjustment from Genesis Block to chain tip: - $ {p} -o +difficulty 0-cur+2016 + $ {p} -o +difficulty 0-cur+{I} Display roughly a block a day over the last two weeks. Note that multiplication is allowed in the nBlocks spec: @@ -144,14 +144,13 @@ EXAMPLES: $ {p} --rpc-backend=aio -H +1000 This program requires a txindex-enabled daemon for correct operation. -""".format( - f = fmt_list(BlocksInfo.fields,fmt='bare'), - s = fmt_list(BlocksInfo.all_stats,fmt='bare'), - p = g.prog_name ) - }, +""" }, 'code': { 'notes': lambda proto,s: s.format( - adj_interval = proto.diff_adjust_interval, + I = proto.diff_adjust_interval, + F = fmt_list(BlocksInfo.fields,fmt='bare'), + S = fmt_list(BlocksInfo.all_stats,fmt='bare'), + p = g.prog_name, ) } } diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 7801f64..ee41377 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -17,6 +17,8 @@ from ..include.common import * from .common import * from .ts_base import * +refdir = os.path.join('test','ref','ticker') + class TestSuiteHelp(TestSuiteBase): 'help, info and usage screens' networks = ('btc','ltc','bch') @@ -50,8 +52,6 @@ class TestSuiteHelp(TestSuiteBase): def longhelpscreens(self): return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') -refdir = os.path.join('test','ref','ticker') - class TestSuiteScripts(TestSuiteBase): 'scripts not requiring a coin daemon' networks = ('btc',) From 11dede26ebdb9dde95284c68493657dd4f853da8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 24 Oct 2022 17:38:12 +0000 Subject: [PATCH 015/108] mmnode-peerblocks: reimplement, add test Testing/demo: $ test/test.py -e main.peerblocks --- mmgen_node_tools/PeerBlocks.py | 155 ++++++++ mmgen_node_tools/PollDisplay.py | 73 ++++ mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_peerblocks.py | 143 +------ setup.cfg | 2 +- test/init.sh | 3 +- .../fakemods/mmgen_node_tools/PeerBlocks.py | 355 ++++++++++++++++++ test/test-release.d/cfg.sh | 13 +- test/test_py_d/cfg.py | 2 + test/test_py_d/ts_main.py | 95 +++++ 10 files changed, 703 insertions(+), 140 deletions(-) create mode 100755 mmgen_node_tools/PeerBlocks.py create mode 100755 mmgen_node_tools/PollDisplay.py create mode 100644 test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py create mode 100755 test/test_py_d/ts_main.py diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py new file mode 100755 index 0000000..a6027ca --- /dev/null +++ b/mmgen_node_tools/PeerBlocks.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen +# https://gitlab.com/mmgen/mmgen + +""" +mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes +""" + +import asyncio +from collections import namedtuple +from mmgen.globalvars import g +from mmgen.util import msg,msg_r,is_int +from mmgen.term import get_term,get_terminal_size,get_char +from mmgen.ui import line_input +from .PollDisplay import PollDisplay + +RED,RESET = ('\033[31m','\033[0m') +COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247,256)) + [231]] +ERASE_ALL,CUR_HOME = ('\033[J','\033[H') +CUR_HIDE,CUR_SHOW = ('\033[?25l','\033[?25h') +term = None + +class Display(PollDisplay): + + poll_secs = 2 + + def __init__(self): + super().__init__() + + global term,term_width + if not term: + term = get_term() + term.init(noecho=True) + term_width = g.columns or get_terminal_size()[0] + msg_r(CUR_HOME+ERASE_ALL+CUR_HOME) + + async def get_info(self,rpc): + return await rpc.call('getpeerinfo') + + def display(self,count): + msg_r( + CUR_HOME + + (ERASE_ALL if count == 1 else '') + + 'CONNECTED PEERS ({a}) {b} - poll {c}'.format( + a = len(self.info), + b = self.desc, + c = count ).ljust(term_width)[:term_width] + + '\n' + + ('\n'.join(self.gen_display()) + '\n' if self.info else '') + + ERASE_ALL + + f"Type a peer number to disconnect, 'q' to quit, or any other key for {self.other_desc} display:" + + '\b' ) + + async def disconnect_node(self,rpc,addr): + return await rpc.call('disconnectnode',addr) + + def get_input(self): + s = get_char(immed_chars='q0123456789',prehold_protect=False,num_chars=1) + if not is_int(s): + return s + with self.info_lock: + msg('') + term.reset() + ret = line_input('peer number> ',insert_txt=s,hold_protect=False) + term.init(noecho=True) + self.enable_display = False # prevent display from updating before process_input() + return ret + + async def process_input(self,rpc): + + ids = tuple(str(i['id']) for i in self.info) + ret = False + msg_r(CUR_HIDE) + + if self.input in ids: + from mmgen.exception import RPCFailure + addr = self.info[ids.index(self.input)]['addr'] + try: + await self.disconnect_node(rpc,addr) + except RPCFailure: + msg_r(f'Unable to disconnect peer {self.input} ({addr})') + else: + msg_r(f'Disconnecting peer {self.input} ({addr})') + await asyncio.sleep(1) + elif self.input and is_int(self.input[0]): + msg_r(f'{self.input}: invalid peer number ') + await asyncio.sleep(0.5) + else: + ret = True + + msg_r(CUR_SHOW) + return ret + +class BlocksDisplay(Display): + + desc = 'Blocks in Flight' + other_desc = 'address' + + def gen_display(self): + + pd = namedtuple('peer_data',['id','blks_data','blks_width']) + bd = namedtuple('block_datum',['num','disp']) + + def gen_block_data(): + global min_height + min_height = None + for d in self.info: + if d.get('inflight'): + blocks = d['inflight'] + min_height = min(blocks) if not min_height else min(min_height,min(blocks)) + line = ' '.join(map(str,blocks))[:blks_field_width] + blocks_disp = line.split() + yield pd( + d['id'], + [bd(blocks[i],blocks_disp[i]) for i in range(len(blocks_disp))], + len(line) ) + else: + yield pd(d['id'],[],0) + + def gen_line(peer_data): + for blk in peer_data.blks_data: + yield (RED if blk.num == min_height else COLORS[blk.num % 10]) + blk.disp + RESET + + id_width = max(2, max(len(str(i['id'])) for i in self.info)) + blks_field_width = term_width - 2 - id_width + fs = '{:>%s}: {}' % id_width + + # we must iterate through all data to get 'min_height' before calling gen_line(): + for peer_data in tuple(gen_block_data()): + yield fs.format( + peer_data.id, + ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width) ) + +class PeersDisplay(Display): + + desc = 'Address Menu' + other_desc = 'blocks' + + def gen_display(self): + id_width = max(2, max(len(str(i['id'])) for i in self.info)) + addr_width = max(len(str(i['addr'])) for i in self.info) + for d in self.info: + yield '{a:>{A}}: {b:{B}} {c}'.format( + a = d['id'], + A = id_width, + b = d['addr'], + B = addr_width, + c = d['subver'] + ).ljust(term_width)[:term_width] diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py new file mode 100755 index 0000000..06b7d7f --- /dev/null +++ b/mmgen_node_tools/PollDisplay.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen +# https://gitlab.com/mmgen/mmgen + +""" +mmgen_node_tools.PollDisplay: update and display RPC data; get input from user +""" + +import sys,threading +from mmgen.util import msg +from mmgen.term import get_char + +class PollDisplay(): + + info = None + input = None + poll_secs = 1 + + def __init__(self): + self.info_lock = threading.Lock() # self.info accessed by 2 threads + self.display_kill_flag = threading.Event() + + def get_input(self): + return get_char(immed_chars='q',prehold_protect=False,num_chars=1) + + async def process_input(self,rpc): + return True + + async def run(self,rpc): + + async def do_display(): + with self.info_lock: + self.info = None + self.input = None + self.enable_display = True + count = 1 + while True: + with self.info_lock: + if self.enable_display: + self.info = await self.get_info(rpc) + self.display(count) + if self.display_kill_flag.wait(self.poll_secs): + self.display_kill_flag.clear() + return + count += 1 + + async def process_input(): + if self.input == None: + sys.exit(1) + elif self.input == 'q': + msg('') + sys.exit(0) + elif self.info: + if await self.process_input(rpc): + return True + else: + return True + + def get_input(): + self.input = self.get_input() + self.display_kill_flag.set() + + while True: + threading.Thread(target=get_input,daemon=True).start() + await do_display() + if await process_input(): + break diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 2d22ff0..1ebddf5 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev8 +3.1.dev9 diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index b775b29..9e6a0ec 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -20,9 +20,8 @@ mmnode-peerblocks: List blocks in flight, disconnect stalling nodes """ -import asyncio -from collections import namedtuple -from mmgen.common import * +from mmgen.opts import init +from mmgen.util import async_run opts_data = { 'text': { @@ -35,120 +34,9 @@ opts_data = { } } -def format_peer_info(peerinfo): - - pd = namedtuple('peer_data',['id','blocks_data','screen_width']) - - def gen_peers(peerinfo): - global min_height - min_height = None - for d in peerinfo: - if 'inflight' in d and d['inflight']: - blocks = d['inflight'] - if not min_height or min_height > blocks[0]: - min_height = blocks[0] - line = ' '.join(map(str,blocks))[:term_width - 2 - id_width] - blocks_disp = line.split() - yield pd( - d['id'], - [(blocks[i],blocks_disp[i]) for i in range(len(blocks_disp))], - len(line) ) - else: - yield pd( d['id'], [], 0 ) - - def gen_line(peer): - if peer.blocks_data: - if peer.blocks_data[0][0] == min_height: - yield RED + peer.blocks_data[0][1] + RESET - peer.blocks_data.pop(0) - for blk,blk_disp in peer.blocks_data: - yield COLORS[blk % 10] + blk_disp + RESET - - id_width = max(2,max(len(str(i['id'])) for i in peerinfo)) - - for peer in tuple(gen_peers(peerinfo)): - line = '{:>{iw}}: {}'.format( - peer.id, - ' '.join(gen_line(peer)), - iw = id_width ) - yield line + ' ' * (term_width - 2 - id_width - peer.screen_width) - -def test_format(): - import json - info = json.loads(open('test_data/peerinfo.json').read()) - print('\n'.join(format_peer_info(info)) + '\n') - sys.exit(0) - -async def inflight_display(rpc): - - count = 1 - while True: - info = await rpc.call('getpeerinfo') - msg_r( - CUR_HOME - + f'ACTIVE PEERS ({len(info)}) Blocks in Flight - poll {count} \n' - + ('\n'.join(format_peer_info(info)) + '\n' if info else '') - + ERASE_ALL + 'Hit ENTER for disconnect menu: ' ) - await asyncio.sleep(2) - count += 1 - -async def do_inflight(rpc): - task = asyncio.ensure_future(inflight_display(rpc)) # Python 3.7+: create_task() - from select import select - - while True: - key = select([sys.stdin], [], [], 0.1)[0] - if key: - sys.stdin.read(1) - task.cancel() - break - await asyncio.sleep(0.1) - - try: - await task - except asyncio.CancelledError: - pass - -async def do_disconnect_menu(rpc): - - while True: - peerinfo = await rpc.call('getpeerinfo') - ids = [str(d['id']) for d in peerinfo] - - msg(f'{CUR_HOME}ACTIVE PEERS ({len(peerinfo)}) Disconnect Menu' + ' '*16) - - def gen_peerinfo(): - for d in peerinfo: - line = f"{d['id']:>{id_width}}: {d['addr']:30} {d['subver']}" - yield line + ' ' * (term_width - len(line)) - - if peerinfo: - id_width = max(2,max(len(str(i['id'])) for i in peerinfo)) - msg('\n'.join(gen_peerinfo())) - - msg_r(ERASE_ALL) - reply = input("Type peer number to disconnect, ENTER to quit menu, 'u' to update peer list: ") - - if reply == '': - return - elif reply == 'u': - msg(f'Updating peer list') - await asyncio.sleep(0.5) - elif reply in ids: - addr = peerinfo[ids.index(reply)]['addr'] - msg(f'Disconnecting peer {reply} ({addr})') - try: - await rpc.call('disconnectnode',addr) - except RPCFailure: - msg(f'Unable to disconnect peer {addr}') - await asyncio.sleep(1.5) - else: - msg(f'{reply!r}: invalid peer number') - await asyncio.sleep(0.5) - async def main(): - msg_r(CUR_HOME+ERASE_ALL) + init(opts_data) from mmgen.protocol import init_proto_from_opts proto = init_proto_from_opts() @@ -156,23 +44,12 @@ async def main(): from mmgen.rpc import rpc_init rpc = await rpc_init(proto) + from .PeerBlocks import BlocksDisplay,PeersDisplay + blocks = BlocksDisplay() + peers = PeersDisplay() + while True: - await do_inflight(rpc) - await do_disconnect_menu(rpc) + await blocks.run(rpc) + await peers.run(rpc) -opts.init(opts_data) - -from mmgen.term import get_terminal_size -term_width = get_terminal_size()[0] - -RED,RESET = ('\033[31m','\033[0m') -COLORS = ['\033[38;5;%s;1m' % c for c in (247,248,249,250,251,252,253,254,255,231)] -ERASE_ALL,ERASE_LINE,CUR_HOME,CUR_HIDE,CUR_SHOW = ( - '\033[J','\033[K','\033[H','\033[?25l','\033[?25h') - -try: - async_run(main()) -except: - from subprocess import run - run(['stty','sane']) - msg('') +async_run(main()) diff --git a/setup.cfg b/setup.cfg index 327cad7..208e03b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev8 + mmgen>=13.3.dev9 packages = mmgen_node_tools diff --git a/test/init.sh b/test/init.sh index 0979c76..a8d03a9 100755 --- a/test/init.sh +++ b/test/init.sh @@ -59,7 +59,8 @@ create_dir_links() { create_test_links() { sources=' test/include - test/overlay + test/overlay/__init__.py + test/overlay/fakemods/mmgen test/__init__.py test/test.py test/unit_tests.py diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py new file mode 100644 index 0000000..0170a2f --- /dev/null +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen +# https://gitlab.com/mmgen/mmgen + +""" +fakemods.mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes - test data +""" + +from .PeerBlocks_orig import * + +class fake_data: + + gp_counter = 0 + dn_counter = 0 + poll_secs = 0.5 + + addresses = """ + 0 3tokm3se4omuhhqgaxiam3yhl474ekbg5xb45kydfno57k7ooapitogv.onion:8333 /Satoshi:24.0.0/ + 1 2rysrq5f2ec4zpnsohyb6mc6l6unosjwaydjefttt34ouogdpqtlo7a5.onion:8333 /Satoshi:0.21.1/ + 2 5bkcnkrihwowji7zwko7ddtpcetz6572zbwdh6aguyt2yz3wgvzqa3ne.onion:8333 /Satoshi:23.0.0/ + 3 ukzy7yvako4z6tvtocsm33yvdxwyx567ioqwfezzewlw2syawzkgrw64.onion:8333 /Satoshi:22.0.0/ + 4 b3r3hhxiauujwj7afmji63forvnd7uhq7ov5x2n7w7hvrhutoq3lhul7.onion:8333 /Satoshi:24.0.0/ + 5 ksownpb4zk4vuyfiowgyvz3kzc2djeiurnnh7pyal3if54n5wzup3afm.onion:8333 /Satoshi:22.0.0/ + 6 xh6vhun75w5y2s2xze2n4rarnduqvwowwthhyiaefly3df2t4guwvjrl.onion:8333 /Satoshi:22.0.0/ + 7 raqbaqcsoldmk7sn6fgoh5bicrlzjdoexz5b2z7jemxb2u6z62vpxto2.onion:8333 /Satoshi:23.0.0/ + 8 5gu3m7jr4tog5tuyisek2fxl5erjgasrxzarnii6cpemmvqhate36hrg.onion:8333 /Satoshi:25.1.0(Aldebaran 3.2.1)/ + 9 ayiyxqgckls4fmzrbbh35ppquhstzhm453xezvzmuoglhnibrxu5ebos.onion:8333 /Satoshi:22.0.0/ + 10 3zmyku3xqrb4mebi2r6l65l5ovmf5yjou2zeywgz5g2ehiqbclppwy6t.onion:8333 /Satoshi:23.0.0/ + 11 ljyfsi4wponyw52o5y75bn6oxktw7kiaes3vdtzf26scapei2ed6yw4m.onion:8333 /Satoshi:0.21.0/ + 12 n5hsupofuhis2xfx7neahntqjl2l3jjp7ait6jsegrp4utj7573qgqsp.onion:8333 /Satoshi:22.0.0/ + 13 klw3dgcuzr4etnkgw5ysnukfl46urhnqrlvjt53v544p32o7acakyps4.onion:8333 /Satoshi:23.0.0/ + 14 icmdqoko4nnqp4aiamiqihbvqlumqrmidyb7n54jdif2uki3qkprwsvd.onion:8333 /Satoshi:24.1.0/ + """ + + # connected peers for each iteration + iterations = """ + 01 0 1 2 3 4 5 6 7 8 + 02 0 1 2 3 4 5 6 7 + 03 0 1 2 3 4 5 6 7 8 + 04 0 2 3 4 5 6 7 8 9 + 05 0 2 3 4 5 6 7 8 9 + 06 0 2 3 4 5 6 7 8 9 10 + 07 0 2 3 4 5 6 7 8 9 10 + 08 0 2 3 4 5 6 7 8 9 10 + 09 0 2 3 4 5 6 7 8 9 10 + 10 0 2 3 4 5 6 7 8 9 10 + 11 2 3 4 5 6 7 8 9 10 + 12 2 3 4 5 6 7 8 9 10 + 13 2 3 4 5 6 7 8 9 10 + 14 2 3 4 5 6 7 8 9 10 + 15 2 3 4 5 6 7 8 9 10 12 + 16 2 3 4 5 6 7 8 9 10 12 + 17 2 3 4 5 6 7 8 9 10 12 + 18 2 3 4 5 6 7 8 9 10 12 + 19 2 3 5 6 7 8 9 10 12 + 20 2 3 5 6 7 8 9 10 12 + 21 2 3 5 6 7 8 9 10 12 + 22 2 3 5 6 7 8 9 10 12 + """ + + # blocks in flight for each iteration for each peer + blocks = { + '0': """ + 01 6917 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 + 02 6917 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 + 03 6917 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 + 04 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 + 05 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 + 06 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 + 07 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 + 08 6918 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 + 09 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 7183 + 10 6920 6937 6939 6942 6944 6946 6947 6950 6951 6953 6971 6976 6985 7083 6913 7183 + """, + '1': """ + 01 6906 6907 6908 6909 6910 6911 6913 6915 6919 6940 6948 6982 6988 7000 7023 7062 + 02 6906 6907 6908 6909 6910 6911 6913 6915 6919 6940 6948 6982 6988 7000 7023 7062 + 03 6908 6909 6910 6911 6913 6915 6919 6940 6948 6982 6988 7000 7023 7062 7088 7107 + """, + '2': """ + 01 6990 6994 6996 6997 6999 7017 7018 7019 7020 7021 7025 7059 7060 7061 7082 7084 + 02 6990 6994 6996 6997 6999 7017 7018 7019 7020 7021 7025 7059 7060 7061 7082 7084 + 03 6997 6999 7017 7018 7019 7020 7021 7025 7059 7060 7061 7082 7084 7085 7089 7108 + 04 7018 7019 7020 7021 7025 7059 7060 7061 7082 7084 7085 7089 7108 7113 6909 7000 + 05 7018 7019 7020 7021 7025 7059 7060 7061 7082 7084 7085 7089 7108 7113 6909 7000 + 06 7020 7021 7025 7059 7060 7061 7082 7084 7085 7089 7108 7113 6909 7000 7112 7142 + 07 7025 7059 7060 7061 7082 7084 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 + 08 7059 7060 7061 7082 7084 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 + 09 7060 7061 7082 7084 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 + 10 7061 7082 7084 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 7201 + 11 7084 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 + 12 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 6985 + 13 7085 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 6985 + 14 7089 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 6985 7217 + 15 7108 7113 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 6985 7217 7245 + 16 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 6985 7217 7245 7250 7261 + 17 6909 7000 7112 7142 7164 7173 7179 7190 7201 7209 6942 6985 7217 7245 7250 7261 + 18 7173 7179 7190 7201 7209 6942 6985 7217 7245 7250 7261 7289 7314 7319 7325 7360 + 19 7179 7190 7201 7209 6942 6985 7217 7245 7250 7261 7289 7314 7319 7325 7360 7379 + 20 7190 7201 7209 6942 6985 7217 7245 7250 7261 7289 7314 7319 7325 7360 7379 7188 + 21 7209 6942 6985 7217 7245 7250 7261 7289 7314 7319 7325 7360 7379 7188 7324 7388 + 22 7209 6942 6985 7217 7245 7250 7261 7289 7314 7319 7325 7360 7379 7188 7324 7388 + """, + '3': """ + 01 6974 6977 6978 6979 6983 6986 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 + 02 6974 6977 6978 6979 6983 6986 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 + 03 6977 6978 6979 6983 6986 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 7106 + 04 6979 6983 6986 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 + 05 6979 6983 6986 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 + 06 6986 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 + 07 6991 6992 6993 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 + 08 6992 6993 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 + 09 6993 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 + 10 6995 6998 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 + 11 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 + 12 7022 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 + 13 7024 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 7214 + 14 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 7214 + 15 7058 7063 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 7214 7223 + 16 7081 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 7214 7223 7247 7257 + 17 7106 7119 6982 7136 7161 7170 7177 7186 7195 6937 6946 7214 7223 7247 7257 7272 + 18 7257 7272 7279 7287 7297 7300 7305 7318 7326 7332 7339 7341 7352 7359 7369 7373 + 19 7279 7287 7297 7300 7305 7318 7326 7332 7339 7341 7352 7359 7369 7373 7381 7383 + 20 7287 7297 7300 7305 7318 7326 7332 7339 7341 7352 7359 7369 7373 7381 7383 7259 + 21 7297 7300 7305 7318 7326 7332 7339 7341 7352 7359 7369 7373 7381 7383 7259 7389 + 22 7297 7300 7305 7318 7326 7332 7339 7341 7352 7359 7369 7373 7381 7383 7259 7389 + """, + '4': """ + 01 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 + 02 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 + 03 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 + 04 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 + 05 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 + 06 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 + 07 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 + 08 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 + 09 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 + 10 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 + 11 7008 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 + 12 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 7083 + 13 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 7083 + 14 7009 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 7083 + 15 7010 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 7083 7225 + 16 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 7083 7225 7259 + 17 7011 7012 7013 7014 7015 7016 7080 7109 6915 7160 7172 7188 7205 7083 7225 7259 + 18 6915 7160 7172 7188 7205 7083 7225 7259 7278 7291 7317 7324 7345 7358 7364 7370 + """, + '5': """ + 01 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 + 02 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 + 03 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7086 7110 + 04 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 + 05 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 + 06 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 7143 + 07 7033 7034 7035 7036 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 7143 7167 + 08 7034 7035 7036 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 7143 7167 7180 + 09 7035 7036 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 7143 7167 7180 7185 + 10 7037 7038 7039 7040 7041 7086 7110 7116 6910 7088 7143 7167 7180 7185 7194 7200 + 11 7041 7086 7110 7116 6910 7088 7143 7167 7180 7185 7194 7200 7206 7207 6939 6944 + 12 7116 6910 7088 7143 7167 7180 7185 7194 7200 7206 7207 6939 6944 6950 6951 6953 + 13 6910 7088 7143 7167 7180 7185 7194 7200 7206 7207 6939 6944 6950 6951 6953 7211 + 14 7088 7143 7167 7180 7185 7194 7200 7206 7207 6939 6944 6950 6951 6953 7211 7220 + 15 7143 7167 7180 7185 7194 7200 7206 7207 6939 6944 6950 6951 6953 7211 7220 7224 + 16 7180 7185 7194 7200 7206 7207 6939 6944 6950 6951 6953 7211 7220 7224 7246 7263 + 17 7185 7194 7200 7206 7207 6939 6944 6950 6951 6953 7211 7220 7224 7246 7263 7268 + 18 7220 7224 7246 7263 7268 7275 7283 7296 7308 7321 7330 7335 7336 7349 7353 7371 + 19 7224 7246 7263 7268 7275 7283 7296 7308 7321 7330 7335 7336 7349 7353 7371 6915 + 20 7224 7246 7263 7268 7275 7283 7296 7308 7321 7330 7335 7336 7349 7353 7371 6915 + 21 7268 7275 7283 7296 7308 7321 7330 7335 7336 7349 7353 7371 6915 7385 7391 7392 + 22 7283 7296 7308 7321 7330 7335 7336 7349 7353 7371 6915 7385 7391 7392 7400 7408 + """, + '6': """ + 01 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 + 02 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 + 03 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7087 + 04 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7087 7115 6940 + 05 7049 7050 7051 7052 7053 7054 7055 7056 7057 7087 7115 6940 + 06 7051 7052 7053 7054 7055 7056 7057 7087 7115 6940 7141 + 07 7052 7053 7054 7055 7056 7057 7087 7115 6940 7141 + 08 7054 7055 7056 7057 7087 7115 6940 7141 7176 + 09 7055 7056 7057 7087 7115 6940 7141 7176 + 10 7056 7057 7087 7115 6940 7141 7176 + 11 7057 7087 7115 6940 7141 7176 + 12 7087 7115 6940 7141 7176 + 13 6940 7141 7176 7213 + 14 7141 7176 7213 + 15 7141 7176 7213 + 16 7141 7176 7213 + 17 7141 7176 7213 + 18 7141 7176 7213 + 19 7176 7213 + 20 7176 7213 + 21 7213 + 22 + """, + '7': """ + 01 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 + 02 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 + 03 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 + 04 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 + 05 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 + 06 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 + 07 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 + 08 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 + 09 7071 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 + 10 7072 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 + 11 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 + 12 7073 7074 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 + 13 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 7210 7212 + 14 7075 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 7210 7212 + 15 7076 7077 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 7210 7212 7226 + 16 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 7210 7212 7226 7254 7265 + 17 7078 7079 7111 7118 6988 7139 7166 7175 7187 7198 6913 7210 7212 7226 7254 7265 + 18 6913 7210 7212 7226 7254 7265 7276 7284 7285 7293 7304 7306 7323 7333 7346 7355 + 19 7212 7226 7254 7265 7276 7284 7285 7293 7304 7306 7323 7333 7346 7355 7375 7083 + 20 7212 7226 7254 7265 7276 7284 7285 7293 7304 7306 7323 7333 7346 7355 7375 7083 + 21 7254 7265 7276 7284 7285 7293 7304 7306 7323 7333 7346 7355 7375 7083 7345 7395 + 22 7276 7284 7285 7293 7304 7306 7323 7333 7346 7355 7375 7083 7345 7395 7396 7405 + """, + '8': """ + 03 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 + 04 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7114 7117 6911 6948 7023 + 05 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7114 7117 6911 6948 7023 + 06 7097 7098 7099 7100 7101 7102 7103 7104 7105 7114 7117 6911 6948 7023 7137 7140 + 07 7099 7100 7101 7102 7103 7104 7105 7114 7117 6911 6948 7023 7137 7140 7162 7169 + 08 7100 7101 7102 7103 7104 7105 7114 7117 6911 6948 7023 7137 7140 7162 7169 7178 + 09 7103 7104 7105 7114 7117 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 + 10 7105 7114 7117 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 7193 7199 + 11 7114 7117 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 7193 7199 6920 + 12 7117 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 7193 7199 6920 6971 + 13 7117 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 7193 7199 6920 6971 + 14 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 7193 7199 6920 6971 7222 + 15 6911 6948 7023 7137 7140 7162 7169 7178 7182 7189 7191 7193 7199 6920 6971 7222 + 16 7162 7169 7178 7182 7189 7191 7193 7199 6920 6971 7222 7248 7249 7256 7260 7264 + 17 7169 7178 7182 7189 7191 7193 7199 6920 6971 7222 7248 7249 7256 7260 7264 7270 + 18 7199 6920 6971 7222 7248 7249 7256 7260 7264 7270 7280 7290 7309 7329 7348 7366 + 19 6971 7222 7248 7249 7256 7260 7264 7270 7280 7290 7309 7329 7348 7366 7380 7172 + 20 6971 7222 7248 7249 7256 7260 7264 7270 7280 7290 7309 7329 7348 7366 7380 7172 + 21 7248 7249 7256 7260 7264 7270 7280 7290 7309 7329 7348 7366 7380 7172 7291 7387 + 22 7260 7264 7270 7280 7290 7309 7329 7348 7366 7380 7172 7291 7387 7399 7402 7406 + """, + '9': """ + 04 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 6919 7062 + 05 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 6919 7062 7107 + 06 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 6919 7062 7107 7138 + 07 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 6919 7062 7107 7138 7168 + 08 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 6919 7062 7107 7138 7168 7181 + 09 7127 7128 7129 7130 7131 7132 7133 7134 7135 6919 7062 7107 7138 7168 7181 7184 + 10 7131 7132 7133 7134 7135 6919 7062 7107 7138 7168 7181 7184 7196 7197 7202 7204 + 11 7132 7133 7134 7135 6919 7062 7107 7138 7168 7181 7184 7196 7197 7202 7204 7208 + 12 7134 7135 6919 7062 7107 7138 7168 7181 7184 7196 7197 7202 7204 7208 6947 6976 + 13 7135 6919 7062 7107 7138 7168 7181 7184 7196 7197 7202 7204 7208 6947 6976 7215 + 14 6919 7062 7107 7138 7168 7181 7184 7196 7197 7202 7204 7208 6947 6976 7215 7219 + 15 7062 7107 7138 7168 7181 7184 7196 7197 7202 7204 7208 6947 6976 7215 7219 7244 + 16 7138 7168 7181 7184 7196 7197 7202 7204 7208 6947 6976 7215 7219 7244 7252 7262 + 17 7168 7181 7184 7196 7197 7202 7204 7208 6947 6976 7215 7219 7244 7252 7262 7269 + 18 7215 7219 7244 7252 7262 7269 7286 7301 7313 7328 7337 7340 7351 7362 7367 7372 + 19 7244 7252 7262 7269 7286 7301 7313 7328 7337 7340 7351 7362 7367 7372 7382 7160 + 20 7252 7262 7269 7286 7301 7313 7328 7337 7340 7351 7362 7367 7372 7382 7160 7205 + 21 7301 7313 7328 7337 7340 7351 7362 7367 7372 7382 7160 7205 7278 7370 7390 7393 + 22 7337 7340 7351 7362 7367 7372 7382 7160 7205 7278 7370 7390 7393 7397 7404 7407 + """, + '10': """ + 06 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 + 07 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 + 08 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 + 09 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 + 10 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 7203 + 11 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 7203 + 12 7151 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 7203 7183 + 13 7152 7153 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 7203 7183 7216 + 14 7154 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 7203 7183 7216 7218 7221 + 15 7155 7156 7157 7158 7159 7163 7165 7171 7174 7192 7203 7183 7216 7218 7221 7227 + 16 7158 7159 7163 7165 7171 7174 7192 7203 7183 7216 7218 7221 7227 7251 7258 7267 + 17 7159 7163 7165 7171 7174 7192 7203 7183 7216 7218 7221 7227 7251 7258 7267 7273 + 18 7277 7282 7288 7294 7302 7312 7320 7327 7331 7338 7347 7354 7357 7361 7365 7368 + 19 7288 7294 7302 7312 7320 7327 7331 7338 7347 7354 7357 7361 7365 7368 7376 7378 + 20 7288 7294 7302 7312 7320 7327 7331 7338 7347 7354 7357 7361 7365 7368 7376 7378 + 21 7302 7312 7320 7327 7331 7338 7347 7354 7357 7361 7365 7368 7376 7378 7364 7394 + 22 7327 7331 7338 7347 7354 7357 7361 7365 7368 7376 7378 7364 7394 7401 7403 7410 + """, + '12': """ + 15 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 + 16 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7253 7266 + 17 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7253 7266 7274 + 18 7281 7292 7298 7299 7303 7307 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 + 19 7299 7303 7307 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 + 20 7303 7307 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 + 21 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 + 22 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 7398 7409 + """ + } + + def make_address_data(): + for line in fake_data.addresses.strip().split('\n'): + data = line.split(maxsplit=2) + yield (data[0], {k:v for k,v in zip(('id','addr','subver'),data)}) + + def make_iterations_data(): + for line in fake_data.iterations.strip().split('\n'): + data = line.split(maxsplit=1) + yield (data[0], data[1].split()) + + def make_blocks_data(iterations): + for peer,blocks_str in fake_data.blocks.items(): + iter_strs = dict([s.lstrip().split(maxsplit=1) for s in blocks_str.strip().split('\n') if ' ' in s]) + yield (peer,dict((i,iter_strs.get(i,'').split()) for i in iterations)) + + def make_data(): + address_data = dict(fake_data.make_address_data()) + iterations_data = dict(fake_data.make_iterations_data()) + blocks_data = dict(fake_data.make_blocks_data(iterations_data)) + + def make_peerinfo(peer_id,blocks,iter_no): + d = address_data[peer_id] + return { + 'id': int(d['id']), + 'addr': d['addr'], + 'subver': d['subver'], + 'inflight': [int(n)+830000 for n in blocks[iter_no]], + } + + def gen_data(): + for iter_no in iterations_data: + yield ( + iter_no, + [make_peerinfo(peer_id,blocks,iter_no) for peer_id,blocks in blocks_data.items() + if peer_id in iterations_data[iter_no] ] + ) + + fake_data.peerinfo = dict(gen_data()) + + async def get_info(self,rpc): + fake_data.gp_counter = (fake_data.gp_counter % 22) + 1 + return fake_data.peerinfo[f'{fake_data.gp_counter:02d}'] + + async def disconnect_node(self,rpc,addr): + from mmgen.exception import RPCFailure + fake_data.dn_counter += 1 + if fake_data.dn_counter % 2: + raise RPCFailure + else: + pass + +fake_data.make_data() + +Display.poll_secs = fake_data.poll_secs +Display.get_info = fake_data.get_info +Display.disconnect_node = fake_data.disconnect_node diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 59db1dd..ba2f6ce 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -14,7 +14,7 @@ # mmnode-feeview - # mmnode-halving-calculator OK # mmnode-netrate - -# mmnode-peerblocks - +# mmnode-peerblocks OK # mmnode-ticker OK # mmnode-txfind - @@ -38,10 +38,10 @@ list_avail_tests() { } init_groups() { - dfl_tests='unit misc scripts btc_rt bch_rt ltc_rt' + dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' extra_tests='' - noalt_tests='unit misc scripts btc_rt' - quick_tests='unit misc scripts btc_rt' + noalt_tests='unit misc scripts btc btc_rt' + quick_tests='unit misc scripts btc btc_rt' qskip_tests='bch_rt ltc_rt' } @@ -61,6 +61,11 @@ init_tests() { t_scripts="- $test_py scripts" f_scripts='No-daemon script tests completed' + i_btc='Bitcoin fake RPC data' + s_btc="The following tests will test various scripts with fake RPC data" + t_btc="- $test_py main" + f_btc='Bitcoin fake RPC data tests completed' + i_btc_rt='Bitcoin regtest' s_btc_rt="The following tests will test various scripts using regtest mode" t_btc_rt="- $test_py regtest" diff --git a/test/test_py_d/cfg.py b/test/test_py_d/cfg.py index d2726c4..c2c72e3 100755 --- a/test/test_py_d/cfg.py +++ b/test/test_py_d/cfg.py @@ -15,6 +15,7 @@ test.test_py_d.cfg: configuration data for test.py import os cmd_groups_dfl = { + 'main': ('TestSuiteMain',{}), 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), 'scripts': ('TestSuiteScripts',{'modname':'misc'}), 'regtest': ('TestSuiteRegtest',{}), @@ -25,6 +26,7 @@ cmd_groups_extra = {} cfgs = { '1': {}, # regtest '2': {}, # scripts + '3': {}, # main } def fixup_cfgs(): pass diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py new file mode 100755 index 0000000..7974c58 --- /dev/null +++ b/test/test_py_d/ts_main.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# +# mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet +# Copyright (C)2013-2022 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen +# https://gitlab.com/mmgen/mmgen + +""" +test_py_d.ts_main: Basic operations tests for the test.py test suite +""" + +import time + +from ..include.common import * +from .common import * +from .ts_base import * + +class TestSuiteMain(TestSuiteBase): + 'basic operations with fake RPC data' + tmpdir_nums = [3] + networks = ('btc',) # fake data, so test peerblocks for BTC mainnet only + passthru_opts = ('daemon_data_dir','rpc_port','coin','testnet','rpc_backend') + segwit_opts_ok = True + color = True + need_daemon = True + + cmd_group_in = ( + ('subgroup.peerblocks', []), + ) + + cmd_subgroups = { + 'peerblocks': ( + "'mmnode-peerblocks' script", + ('peerblocks1', '--help'), + ('peerblocks2', 'interactive'), + ('peerblocks3', 'interactive, 80 columns'), + ), + } + + def peerblocks(self,args,expect_list=None): + t = self.spawn( + f'mmnode-peerblocks', + args ) + if opt.exact_output: # disable echoing of input + t.p.logfile = None + t.p.logfile_read = sys.stdout + if expect_list: + t.match_expect_list(expect_list) + return t + + def peerblocks1(self): + t = self.peerblocks(['--help']) + if opt.pexpect_spawn: + t.send('q') + return t + + def peerblocks2(self,args=[]): + + t = self.peerblocks(args) + + for i in range(5): + t.expect('PEERS') + + t.send('x') + + for i in range(3): + t.expect('PEERS') + + t.send('0') + time.sleep(0.2) + t.send('\n') + t.expect('Unable to disconnect peer 0') + t.expect('PEERS') + + t.send('1') + time.sleep(0.2) + t.send('1\n') + t.expect('11: invalid peer number') + t.expect('PEERS') + + t.send('2') + time.sleep(0.2) + t.send('\n') + t.expect('Disconnecting peer 2') + t.expect('PEERS') + + t.send('q') + + return t + + def peerblocks3(self): + return self.peerblocks2(['--columns=80']) From b640dea1117b7dc2a2f8dd47778ed1919bcc3c04 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 25 Oct 2022 09:10:15 +0000 Subject: [PATCH 016/108] update for MMGen v13.3.dev11 --- mmgen_node_tools/PeerBlocks.py | 1 + mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/test-release.d/cfg.sh | 57 ++++++++++------------------------ test/test_py_d/ts_main.py | 6 ++-- 5 files changed, 23 insertions(+), 45 deletions(-) diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index a6027ca..e38af23 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -67,6 +67,7 @@ class Display(PollDisplay): with self.info_lock: msg('') term.reset() + # readline required for correct operation here; without it, user must re-type first digit ret = line_input('peer number> ',insert_txt=s,hold_protect=False) term.init(noecho=True) self.enable_display = False # prevent display from updating before process_input() diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1ebddf5..e2195ce 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev9 +3.1.dev10 diff --git a/setup.cfg b/setup.cfg index 208e03b..0be313f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev9 + mmgen>=13.3.dev11 packages = mmgen_node_tools diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index ba2f6ce..866f453 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -18,27 +18,18 @@ # mmnode-ticker OK # mmnode-txfind - -list_avail_tests() { - echo "AVAILABLE TESTS:" - echo " unit - unit tests" - echo " btc_rt - Bitcoin regtest" - echo " bch_rt - Bitcoin Cash Node (BCH) regtest" - echo " ltc_rt - Litecoin regtest" - echo " scripts - tests of scripts not requiring a coin daemon" - echo " misc - miscellaneous tests that don't fit in the above categories" - echo - echo "AVAILABLE TEST GROUPS:" - echo " default - All tests minus the extra tests" - echo " extra - All tests minus the default tests" - echo " noalt - BTC-only tests" - echo " quick - Default tests minus btc_tn, bch, bch_rt, ltc and ltc_rt" - echo " qskip - The tests skipped in the 'quick' test group" - echo - echo "By default, all tests are run" -} +all_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + +groups_desc=" + default - All tests minus the extra tests + extra - All tests minus the default tests + noalt - BTC-only tests + quick - Default tests minus btc_tn, bch, bch_rt, ltc and ltc_rt + qskip - The tests skipped in the 'quick' test group +" init_groups() { - dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + dfl_tests=$all_tests extra_tests='' noalt_tests='unit misc scripts btc btc_rt' quick_tests='unit misc scripts btc btc_rt' @@ -46,38 +37,24 @@ init_groups() { } init_tests() { - i_unit='Unit' - s_unit="The following tests will test various low-level subsystems" + d_unit="low-level subsystems" t_unit="- $unit_tests_py" - f_unit='Unit tests completed' - i_misc='Misc' - s_misc="The following tests will test miscellaneous script features" + d_misc="miscellaneous features" t_misc="- $test_py helpscreens" - f_misc='Misc tests completed' - i_scripts='No-daemon scripts' - s_scripts="The following tests will test scripts not requiring a coin daemon" + d_scripts="scripts not requiring a coin daemon" t_scripts="- $test_py scripts" - f_scripts='No-daemon script tests completed' - i_btc='Bitcoin fake RPC data' - s_btc="The following tests will test various scripts with fake RPC data" + d_btc="Bitcoin with emulated RPC data" t_btc="- $test_py main" - f_btc='Bitcoin fake RPC data tests completed' - i_btc_rt='Bitcoin regtest' - s_btc_rt="The following tests will test various scripts using regtest mode" + d_btc_rt="Bitcoin regtest" t_btc_rt="- $test_py regtest" - f_btc_rt='Regtest mode tests for BTC completed' - i_bch_rt='BitcoinCashNode (BCH) regtest' - s_bch_rt="The following tests will test various scripts using regtest mode" + d_bch_rt="Bitcoin Cash Node (BCH) regtest" t_bch_rt="- $test_py --coin=bch regtest" - f_bch_rt='Regtest mode tests for BCH completed' - i_ltc_rt='Litecoin regtest' - s_ltc_rt="The following tests will test various scripts using regtest mode" + d_ltc_rt="Litecoin regtest" t_ltc_rt="- $test_py --coin=ltc regtest" - f_ltc_rt='Regtest mode tests for LTC completed' } diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index 7974c58..aa175cc 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -71,19 +71,19 @@ class TestSuiteMain(TestSuiteBase): t.send('0') time.sleep(0.2) - t.send('\n') + t.send('\n' if opt.pexpect_spawn else '0\n') # TODO: check for readline availability t.expect('Unable to disconnect peer 0') t.expect('PEERS') t.send('1') time.sleep(0.2) - t.send('1\n') + t.send('1\n' if opt.pexpect_spawn else '11\n') t.expect('11: invalid peer number') t.expect('PEERS') t.send('2') time.sleep(0.2) - t.send('\n') + t.send('\n' if opt.pexpect_spawn else '2\n') t.expect('Disconnecting peer 2') t.expect('PEERS') From 6ad22bdf5c5c92813313bbaeff1e080775f12bdf Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 27 Oct 2022 16:47:22 +0000 Subject: [PATCH 017/108] various fixes and cleanups --- mmgen_node_tools/PeerBlocks.py | 2 +- mmgen_node_tools/Term.py | 49 ------------------- mmgen_node_tools/main_feeview.py | 46 ++++++----------- .../fakemods/mmgen_node_tools/PeerBlocks.py | 37 +++++++------- test/test-release.d/cfg.sh | 2 +- test/test_py_d/ts_regtest.py | 2 + 6 files changed, 38 insertions(+), 100 deletions(-) delete mode 100755 mmgen_node_tools/Term.py diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index e38af23..4a4a940 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -37,7 +37,7 @@ class Display(PollDisplay): if not term: term = get_term() term.init(noecho=True) - term_width = g.columns or get_terminal_size()[0] + term_width = g.columns or get_terminal_size().width msg_r(CUR_HOME+ERASE_ALL+CUR_HOME) async def get_info(self,rpc): diff --git a/mmgen_node_tools/Term.py b/mmgen_node_tools/Term.py deleted file mode 100755 index 7b5b7e0..0000000 --- a/mmgen_node_tools/Term.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -# -# mmgen = Multi-Mode GENerator, command-line Bitcoin cold storage solution -# Copyright (C)2013-2016 Philemon -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -mmgen_node_tools.Term: terminal routines for MMGen node tools -""" - -import sys,os,termios - -def get_keypress(prompt="",esc_sequences=False): - - import time,tty,select - sys.stderr.write(prompt) - - fd = sys.stdin.fileno() -# old = termios.tcgetattr(fd) # see below - tty.setcbreak(fd) # must do this, even if it was set at program launch - - def osread_chk(n): - while True: - try: - return os.read(fd,n) - except: - time.sleep(0.1) - - # Must use os.read() for unbuffered read, otherwise select() will never return true - s = osread_chk(1) - if esc_sequences: - if s == '\x1b': - if select.select([sys.stdin],[],[],0)[0]: - s += osread_chk(2) - -# Leave the term in cbreak mode, restore at exit -# termios.tcsetattr(fd, termios.TCSADRAIN, old) - return s diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index e5f2f00..fd6d7d3 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -61,7 +61,6 @@ opts.init({ -P, --pager Pipe the output to a pager -r, --ranges Display fee brackets as ranges -s, --show-mb-col Display column with each fee bracket’s megabyte count --w, --width=W Force output width of 'W' columns (default: term width) """, 'notes': """ + By default, fee bracket row labels include only the top of the range. @@ -83,16 +82,12 @@ if opt.ignore_below: die(1,'Conflicting options: --ignore-below, --show-empty') ignore_below = parse_bytespec(opt.ignore_below) -if opt.precision: - precision = check_int_between(opt.precision,min_prec,max_prec,'--precision arg') -else: - precision = dfl_prec +precision = ( + check_int_between(opt.precision,min_prec,max_prec,'--precision arg') + if opt.precision else dfl_prec ) -if opt.width: - width = check_int_between(opt.width,40,1024,'--width arg') -else: - from mmgen.term import get_terminal_size - width = get_terminal_size()[0] +from mmgen.term import get_terminal_size +width = g.columns or get_terminal_size().width class fee_bracket: def __init__(self,top,bottom): @@ -102,12 +97,6 @@ class fee_bracket: self.tx_bytes_cum = 0 self.skip = False -def get_fake_data(fn): # for debugging - import json - from mmgen.rpc import json_encoder - from decimal import Decimal - return json.loads(open(os.path.join(fn)).read(),parse_float=Decimal) - def log(data,fn): import json from mmgen.rpc import json_encoder @@ -129,9 +118,6 @@ def create_data(coin_amt,mempool): while out and out[-1].tx_bytes == 0: out.pop() - if not out: - die(1,'No data!') - out.reverse() # cumulative totals and display are top-down # calculate cumulative byte totals, filter rows: @@ -152,6 +138,7 @@ def gen_header(host,blockcount): make_timestr(), blockcount, )) + if opt.show_empty: yield('Displaying all fee brackets') elif opt.ignore_below: @@ -159,29 +146,28 @@ def gen_header(host,blockcount): ignore_below, int2bytespec(ignore_below,'MB','0.6'), )) + if opt.include_current: yield('Including transactions in current fee bracket in Total MB amounts') def fmt_mb(n): - return int2bytespec(n,'MB',f'0.{precision}',False) + return int2bytespec(n,'MB',f'0.{precision}',print_sym=False) def gen_body(data): - tx_bytes_max = max(i.tx_bytes for i in data) - top_max = max(i.top for i in data if not i.skip) - bot_max = max(i.bottom for i in data if not i.skip) + tx_bytes_max = max((i.tx_bytes for i in data),default=0) + top_max = max((i.top for i in data),default=0) + bot_max = max((i.bottom for i in data),default=0) col1_w = max(len(f'{bot_max}-{top_max}') if opt.ranges else len(f'{top_max}'),6) col2_w = len(fmt_mb(tx_bytes_max)) if opt.show_mb_col else 0 - col3_w = len(fmt_mb(data[-1].tx_bytes_cum)) + col3_w = len(fmt_mb(data[-1].tx_bytes_cum)) if data else 0 col4_w = width - col1_w - col2_w - col3_w - (4 if col2_w else 3) if opt.show_mb_col: fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w,col2_w,col3_w) else: fs = '{a:<%i} {c:>%i} {d}' % (col1_w,col3_w) - yield( - '\n' + fs.format(a='', b='', c=f'{"Total":<{col3_w}}', d='') + - '\n' + fs.format(a='sat/B', b=f'{"MB":<{col2_w}}', c=f'{"MB":<{col3_w}}', d='') - ) + yield fs.format(a='', b='', c=f'{"Total":<{col3_w}}', d='') + yield fs.format(a='sat/B', b=f'{"MB":<{col2_w}}', c=f'{"MB":<{col3_w}}', d='') for i in data: if not i.skip: @@ -195,7 +181,7 @@ def gen_body(data): yield(fs.format( a = 'TOTAL', b = '', - c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes), + c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes if data else 0), d = '' )) async def main(): @@ -206,9 +192,7 @@ async def main(): from mmgen.rpc import rpc_init c = await rpc_init(proto) -# pmsg(await c.call('getmempoolinfo')) mempool = await c.call('getrawmempool',True) -# mempool = get_fake_data('test_data/mempool-sample.json') if opt.log: log(mempool,'mempool.json') diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py index 0170a2f..2484628 100644 --- a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -297,25 +297,22 @@ class fake_data: """ } - def make_address_data(): - for line in fake_data.addresses.strip().split('\n'): - data = line.split(maxsplit=2) - yield (data[0], {k:v for k,v in zip(('id','addr','subver'),data)}) - - def make_iterations_data(): - for line in fake_data.iterations.strip().split('\n'): - data = line.split(maxsplit=1) - yield (data[0], data[1].split()) - - def make_blocks_data(iterations): - for peer,blocks_str in fake_data.blocks.items(): - iter_strs = dict([s.lstrip().split(maxsplit=1) for s in blocks_str.strip().split('\n') if ' ' in s]) - yield (peer,dict((i,iter_strs.get(i,'').split()) for i in iterations)) - def make_data(): - address_data = dict(fake_data.make_address_data()) - iterations_data = dict(fake_data.make_iterations_data()) - blocks_data = dict(fake_data.make_blocks_data(iterations_data)) + + def gen_address_data(): + for line in fake_data.addresses.strip().split('\n'): + data = line.split(maxsplit=2) + yield (data[0], {k:v for k,v in zip(('id','addr','subver'),data)}) + + def gen_iterations_data(): + for line in fake_data.iterations.strip().split('\n'): + data = line.split(maxsplit=1) + yield (data[0], data[1].split()) + + def gen_blocks_data(iterations): + for peer,blocks_str in fake_data.blocks.items(): + iter_strs = dict([s.lstrip().split(maxsplit=1) for s in blocks_str.strip().split('\n') if ' ' in s]) + yield (peer,dict((i,iter_strs.get(i,'').split()) for i in iterations)) def make_peerinfo(peer_id,blocks,iter_no): d = address_data[peer_id] @@ -334,6 +331,10 @@ class fake_data: if peer_id in iterations_data[iter_no] ] ) + address_data = dict(gen_address_data()) + iterations_data = dict(gen_iterations_data()) + blocks_data = dict(gen_blocks_data(iterations_data)) + fake_data.peerinfo = dict(gen_data()) async def get_info(self,rpc): diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 866f453..e277ab6 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -24,7 +24,7 @@ groups_desc=" default - All tests minus the extra tests extra - All tests minus the default tests noalt - BTC-only tests - quick - Default tests minus btc_tn, bch, bch_rt, ltc and ltc_rt + quick - Default tests minus bch_rt and ltc_rt qskip - The tests skipped in the 'quick' test group " diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index d6b037e..737cb75 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -17,6 +17,7 @@ from mmgen.globalvars import g from mmgen.opts import opt from mmgen.util import die,gmsg from mmgen.protocol import init_proto +from mmgen.proto.btc.regtest import MMGenRegtest from ..include.common import * from .common import * @@ -93,6 +94,7 @@ class TestSuiteRegtest(TestSuiteBase): die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto(self.proto.coin,network='regtest',need_amt=True) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) + self.regtest = MMGenRegtest(self.proto.coin) def setup(self): stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) From a8a44ab646e6131c78a766de548c55b7fcab12d8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 27 Oct 2022 16:47:22 +0000 Subject: [PATCH 018/108] mmnode-feeview: display improvements, support BCH, add test Testing/demo: $ test/test.py -e regtest.feeview $ test/test.py -e --coin=ltc regtest.feeview $ test/test.py -e --coin=bch regtest.feeview --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 43 ++++++---- setup.cfg | 2 +- test/test-release.d/cfg.sh | 2 +- test/test_py_d/ts_regtest.py | 135 +++++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index e2195ce..9ddd38f 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev10 +3.1.dev11 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index fd6d7d3..ce1fbdb 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -52,7 +52,8 @@ opts.init({ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -c, --include-current Include current bracket’s TXs in cumulative MB value --d, --detail Same as --ranges --show-mb-col --precision=6 +-d, --outdir=D Write log data to directory 'D' +-D, --detail Same as --ranges --show-mb-col --precision=6 -e, --show-empty Show all fee brackets, including empty ones -i, --ignore-below=B Ignore fee brackets with less than 'B' bytes of TXs -l, --log Log JSON-RPC mempool data to 'mempool.json' @@ -100,18 +101,24 @@ class fee_bracket: def log(data,fn): import json from mmgen.rpc import json_encoder - open(fn,'w').write(json.dumps(data,cls=json_encoder,sort_keys=True,indent=4)) + from mmgen.fileutil import write_data_to_file + write_data_to_file( + outfile = fn, + data = json.dumps(data,cls=json_encoder,sort_keys=True,indent=4), + desc = 'mempool', + ask_overwrite = False ) def create_data(coin_amt,mempool): out = [fee_bracket(fee_brackets[i],fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] # populate fee brackets: + size_key = 'size' if proto.coin == 'BCH' else 'vsize' for tx in mempool.values(): fee = coin_amt(tx['fees']['base']).to_unit('satoshi') - vsize = tx['vsize'] + size = tx[size_key] for bracket in out: - if fee / vsize < bracket.top: - bracket.tx_bytes += vsize + if fee / size < bracket.top: + bracket.tx_bytes += size break # remove empty top brackets: @@ -132,19 +139,23 @@ def create_data(coin_amt,mempool): return out -def gen_header(host,blockcount): - yield('MEMPOOL FEE STRUCTURE ({})\n{} UTC\nBlock {}'.format( - host, - make_timestr(), - blockcount, - )) +def gen_header(host,mempool,blockcount): + + yield(fmt(f""" + Mempool Fee Structure + Date: {make_timestr()} UTC + Host: {host} + Network: {proto.coin.upper()} {proto.network.upper()} + Block: {blockcount} + TX count: {len(mempool)} + """)).strip() if opt.show_empty: yield('Displaying all fee brackets') elif opt.ignore_below: - yield('Ignoring fee brackets with less than {} bytes ({})'.format( + yield('Ignoring fee brackets with less than {:,} bytes ({})'.format( ignore_below, - int2bytespec(ignore_below,'MB','0.6'), + int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), )) if opt.include_current: @@ -187,6 +198,7 @@ def gen_body(data): async def main(): from mmgen.protocol import init_proto_from_opts + global proto proto = init_proto_from_opts(need_amt=True) from mmgen.rpc import rpc_init @@ -199,7 +211,10 @@ async def main(): data = create_data(proto.coin_amt,mempool) stdout_or_pager( - '\n'.join(gen_header(c.host,await c.call('getblockcount'))) + '\n' + + '\n'.join(gen_header( + c.host, + mempool, + await c.call('getblockcount') )) + '\n\n' + '\n'.join(gen_body(data)) + '\n' ) async_run(main()) diff --git a/setup.cfg b/setup.cfg index 0be313f..3570d34 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev11 + mmgen>=13.3.dev12 packages = mmgen_node_tools diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index e277ab6..cfe98b7 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -11,7 +11,7 @@ # Testing status # mmnode-addrbal OK # mmnode-blocks-info OK -# mmnode-feeview - +# mmnode-feeview OK # mmnode-halving-calculator OK # mmnode-netrate - # mmnode-peerblocks OK diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 737cb75..2501160 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -47,6 +47,7 @@ class TestSuiteRegtest(TestSuiteBase): ('subgroup.fund_addrbal', []), ('subgroup.addrbal', ['fund_addrbal']), ('subgroup.blocks_info', ['addrbal']), + ('subgroup.feeview', []), ('stop', 'stopping regtest daemon'), ) cmd_subgroups = { @@ -84,6 +85,18 @@ class TestSuiteRegtest(TestSuiteBase): ('blocks_info3', "blocks-info +100"), ('blocks_info4', "blocks-info --miner-info --fields=all --stats=all +1"), ), + 'feeview': ( + "'mmnode-feeview' script", + ('feeview_setup', 'setting up feeview test'), + ('feeview1', "'mmnode-feeview'"), + ('feeview2', "'mmnode-feeview --columns=40 --include-current'"), + ('feeview3', "'mmnode-feeview --precision=6'"), + ('feeview4', "'mmnode-feeview --detail'"), + ('feeview5', "'mmnode-feeview --show-empty --log'"), + ('feeview6', "'mmnode-feeview --ignore-below=1MB'"), + ('feeview7', "'mmnode-feeview --ignore-below=20kB'"), + ('feeview8', "'mmnode-feeview' (empty mempool)"), + ), } def __init__(self,trunner,cfgs,spawn): @@ -247,6 +260,128 @@ class TestSuiteRegtest(TestSuiteBase): 'Next diff adjust: 2016' ]) + async def feeview_setup(self): + + def create_pairs(nPairs): + + from mmgen.tool.api import tool_api + from collections import namedtuple + + t = tool_api() + t.init_coin(self.proto.coin,self.proto.network) + t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32' + wp = namedtuple('wifaddrpair',['wif','addr']) + + def gen(): + for n in range(1,nPairs+1): + wif = t.hex2wif(f'{n:064x}') + yield wp( wif, t.wif2addr(wif) ) + + return list(gen()) + + def gen_fees(n_in,low,high): + + # very approximate tx size estimation: + ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) + x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt(self.proto.coin_amt.satoshi) + + n = n_in - 1 + vmax = high - low + + for i in range(n_in): + yield (low + (i/n)**6 * vmax) * x + + async def do_tx(inputs,outputs,wif): + tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) + tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) + assert tx['complete'] == True + return tx['hex'] + + async def do_tx1(): + us = await r.rpc_call('listunspent',wallet='miner') + fee = self.proto.coin_amt('0.001') + outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} + outputs.update({burn_addr: us[0]['amount'] - (tx1_amt*nTxs) - fee}) + return await do_tx( + [{ 'txid': us[0]['txid'], 'vout': 0 }], + outputs, + r.miner_wif ) + + async def do_tx2(tx,pairno): + fee = fees[pairno] + outputs = {p.addr:tx2_amt for p in pairs} + outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) + return await do_tx( + [{ 'txid': tx['txid'], 'vout': pairno }], + outputs, + pairs[pairno].wif ) + + async def do_txs(tx_in): + for pairno in range(nTxs): + tx_hex = await do_tx2(tx_in,pairno) + await r.rpc_call('sendrawtransaction',tx_hex) + + self.spawn('',msg_only=True) + + r = self.regtest + nPairs = 100 + nTxs = 25 + tx1_amt = self.proto.coin_amt('{:0.4f}'.format(24 / (nTxs+1))) # 25 BTC subsidy + tx2_amt = self.proto.coin_amt('0.0001') + + imsg(f'Creating {nPairs} key-address pairs') + pairs = create_pairs(nPairs+1) + burn_addr = pairs.pop()[1] + + imsg(f'Creating funding transaction with {nTxs} outputs of value {tx1_amt} {self.proto.coin}') + tx1_hex = await do_tx1() + + imsg(f'Relaying funding transaction') + await r.rpc_call('sendrawtransaction',tx1_hex) + + imsg(f'Mining a block') + await r.generate(1,silent=True) + + imsg(f'Generating fees for mempool transactions') + fees = list(gen_fees(nTxs,2,120)) + + imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') + await do_txs(await r.rpc_call('decoderawtransaction',tx1_hex)) + + return 'ok' + + def _feeview(self,args,expect_list=[]): + t = self.spawn('mmnode-feeview',args) + if expect_list: + t.match_expect_list(expect_list) + return t + + def feeview1(self): + return self._feeview([]) + + def feeview2(self): + return self._feeview(['--columns=40','--include-current']) + + def feeview3(self): + return self._feeview(['--precision=6']) + + def feeview4(self): + return self._feeview(['--detail']) + + def feeview5(self): + return self._feeview(['--show-empty','--log',f'--outdir={self.tmpdir}']) + + def feeview6(self): + return self._feeview(['--ignore-below=1MB']) + + def feeview7(self): + return self._feeview(['--ignore-below=4kB']) + + async def feeview8(self): + imsg('Clearing mempool') + await self.regtest.generate(1,silent=True) + return self._feeview([]) + def stop(self): if opt.no_daemon_stop: self.spawn('',msg_only=True) From 3a972f2149e4fddf24c0cc0d7dfa4e4c75fa7c1a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 28 Oct 2022 11:33:52 +0000 Subject: [PATCH 019/108] update for MMGen v13.3.dev13 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/test_py_d/ts_main.py | 31 +++++++++++++++++-------------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 9ddd38f..4913611 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev11 +3.1.dev12 diff --git a/setup.cfg b/setup.cfg index 3570d34..5c13b13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev12 + mmgen>=13.3.dev13 packages = mmgen_node_tools diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index aa175cc..171ffc5 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -35,15 +35,16 @@ class TestSuiteMain(TestSuiteBase): 'peerblocks': ( "'mmnode-peerblocks' script", ('peerblocks1', '--help'), - ('peerblocks2', 'interactive'), - ('peerblocks3', 'interactive, 80 columns'), + ('peerblocks2', 'interactive (popen spawn)'), + ('peerblocks3', 'interactive, 80 columns (pexpect_spawn)'), ), } - def peerblocks(self,args,expect_list=None): + def peerblocks(self,args,expect_list=None,pexpect_spawn=False): t = self.spawn( f'mmnode-peerblocks', - args ) + args, + pexpect_spawn = pexpect_spawn ) if opt.exact_output: # disable echoing of input t.p.logfile = None t.p.logfile_read = sys.stdout @@ -53,13 +54,13 @@ class TestSuiteMain(TestSuiteBase): def peerblocks1(self): t = self.peerblocks(['--help']) - if opt.pexpect_spawn: + if t.pexpect_spawn: t.send('q') return t - def peerblocks2(self,args=[]): + def peerblocks2(self,args=[],pexpect_spawn=False): - t = self.peerblocks(args) + t = self.peerblocks(args,pexpect_spawn=pexpect_spawn) for i in range(5): t.expect('PEERS') @@ -69,21 +70,23 @@ class TestSuiteMain(TestSuiteBase): for i in range(3): t.expect('PEERS') + sleep_secs = 0.2 + t.send('0') - time.sleep(0.2) - t.send('\n' if opt.pexpect_spawn else '0\n') # TODO: check for readline availability + time.sleep(sleep_secs) + t.send('\n' if pexpect_spawn else '0\n') # TODO: check for readline availability t.expect('Unable to disconnect peer 0') t.expect('PEERS') t.send('1') - time.sleep(0.2) - t.send('1\n' if opt.pexpect_spawn else '11\n') + time.sleep(sleep_secs) + t.send('1\n' if pexpect_spawn else '11\n') t.expect('11: invalid peer number') t.expect('PEERS') t.send('2') - time.sleep(0.2) - t.send('\n' if opt.pexpect_spawn else '2\n') + time.sleep(sleep_secs) + t.send('\n' if pexpect_spawn else '2\n') t.expect('Disconnecting peer 2') t.expect('PEERS') @@ -92,4 +95,4 @@ class TestSuiteMain(TestSuiteBase): return t def peerblocks3(self): - return self.peerblocks2(['--columns=80']) + return self.peerblocks2(['--columns=80'],pexpect_spawn=True) From c53e7cc65f5d4a8a14e391461d04071df711392c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 29 Oct 2022 20:12:39 +0000 Subject: [PATCH 020/108] update for MMGen v13.3.dev14 --- mmgen_node_tools/PeerBlocks.py | 2 +- mmgen_node_tools/PollDisplay.py | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index 4a4a940..899f962 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -61,7 +61,7 @@ class Display(PollDisplay): return await rpc.call('disconnectnode',addr) def get_input(self): - s = get_char(immed_chars='q0123456789',prehold_protect=False,num_chars=1) + s = get_char(immed_chars='q0123456789',prehold_protect=False,num_bytes=1) if not is_int(s): return s with self.info_lock: diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 06b7d7f..bff4683 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -27,7 +27,7 @@ class PollDisplay(): self.display_kill_flag = threading.Event() def get_input(self): - return get_char(immed_chars='q',prehold_protect=False,num_chars=1) + return get_char(immed_chars='q',prehold_protect=False,num_bytes=1) async def process_input(self,rpc): return True diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4913611..102a890 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev12 +3.1.dev13 diff --git a/setup.cfg b/setup.cfg index 5c13b13..e9a2587 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev13 + mmgen>=13.3.dev14 packages = mmgen_node_tools From affa8910413a84cd77b4683be85c2b206ff7c2cf Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 13 Nov 2022 15:45:57 +0000 Subject: [PATCH 021/108] update for MMGen v13.3.dev20 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 4 ++-- mmgen_node_tools/main_halving_calculator.py | 6 +++--- setup.cfg | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 102a890..1bc37c8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev13 +3.1.dev14 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 4218f6a..b0ab4f8 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -55,7 +55,7 @@ def do_output(proto,addr_data,blk_hdrs): make_timestr( blk_hdrs[u['height']]['time'] ), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(color=True,fs='6.8') + proto.coin_amt(u['amount']).fmt(color=True,iwidth=6,prec=8) )) else: Msg(f'{indent}No balance') @@ -94,7 +94,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = proto.coin_amt(sum(u['amount'] for u in unspents)).fmt(color=True,fs='7.8') + A = proto.coin_amt(sum(u['amount'] for u in unspents)).fmt(color=True,iwidth=7,prec=8) )) else: Msg(fs.format( diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index c804d8f..5cbf2b8 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -168,9 +168,9 @@ async def main(): c = date(t), d = ' P' if n > nhist else '' if n < nhist else ' E', e = f'{bdr:8.5f}', - f = proto.coin_amt(sub,from_unit='satoshi').fmt(fs='2.8'), - g = proto.coin_amt(mined,from_unit='satoshi').fmt(fs='8.8'), - h = proto.coin_amt(total_mined,from_unit='satoshi').fmt(fs='8.8') + f = proto.coin_amt(sub,from_unit='satoshi').fmt(iwidth=2,prec=8), + g = proto.coin_amt(mined,from_unit='satoshi').fmt(iwidth=8,prec=8), + h = proto.coin_amt(total_mined,from_unit='satoshi').fmt(iwidth=8,prec=8) ) for n,sub,blk,mined,total_mined,bdr,t in gen_data()) ) diff --git a/setup.cfg b/setup.cfg index e9a2587..65b9639 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev14 + mmgen>=13.3.dev20 packages = mmgen_node_tools From eecf8acd0d8ece32be7b046e80964cb84a5a0931 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 28 Mar 2023 18:16:33 +0000 Subject: [PATCH 022/108] minor fixes and cleanups --- mmgen_node_tools/PeerBlocks.py | 1 + mmgen_node_tools/main_txfind.py | 1 + test/test_py_d/ts_misc.py | 2 +- test/test_py_d/ts_regtest.py | 13 +++++++------ test/unit_tests_d/ut_BlocksInfo.py | 8 +++++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index 899f962..6df9894 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -31,6 +31,7 @@ class Display(PollDisplay): poll_secs = 2 def __init__(self): + super().__init__() global term,term_width diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 2672a11..d0a04e2 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -86,6 +86,7 @@ async def main(txid): return exitval cmd_args = opts.init(opts_data) + msgs = msg_data['quiet' if opt.quiet else 'normal'] if len(cmd_args) != 1: diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index ee41377..cc4c22d 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -ts_misc.py: Miscellaneous test groups for the test.py test suite +test.test_py_d.ts_misc: Miscellaneous test groups for the test.py test suite """ import shutil diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 2501160..fd19099 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -ts_regtest.py: Regtest tests for the test.py test suite +test.test_py_d.ts_regtest: Regtest tests for the test.py test suite """ import os @@ -273,7 +273,7 @@ class TestSuiteRegtest(TestSuiteBase): wp = namedtuple('wifaddrpair',['wif','addr']) def gen(): - for n in range(1,nPairs+1): + for n in range(0xfaceface,nPairs+0xfaceface): wif = t.hex2wif(f'{n:064x}') yield wp( wif, t.wif2addr(wif) ) @@ -299,11 +299,12 @@ class TestSuiteRegtest(TestSuiteBase): async def do_tx1(): us = await r.rpc_call('listunspent',wallet='miner') + tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} - outputs.update({burn_addr: us[0]['amount'] - (tx1_amt*nTxs) - fee}) + outputs.update({burn_addr: tx_input['amount'] - (tx1_amt*nTxs) - fee}) return await do_tx( - [{ 'txid': us[0]['txid'], 'vout': 0 }], + [{ 'txid': tx_input['txid'], 'vout': 0 }], outputs, r.miner_wif ) @@ -326,8 +327,8 @@ class TestSuiteRegtest(TestSuiteBase): r = self.regtest nPairs = 100 nTxs = 25 - tx1_amt = self.proto.coin_amt('{:0.4f}'.format(24 / (nTxs+1))) # 25 BTC subsidy - tx2_amt = self.proto.coin_amt('0.0001') + tx1_amt = self.proto.coin_amt('{:0.4f}'.format(24 / nTxs)) # 25 BTC subsidy, leave extra for fee + tx2_amt = self.proto.coin_amt('0.00005') # make this as small as possible imsg(f'Creating {nPairs} key-address pairs') pairs = create_pairs(nPairs+1) diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/unit_tests_d/ut_BlocksInfo.py index e83cca0..ab09882 100755 --- a/test/unit_tests_d/ut_BlocksInfo.py +++ b/test/unit_tests_d/ut_BlocksInfo.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """ -test.unit_tests_d.nt_BlocksInfo: - BlocksInfo unit test for the MMGen Node Tools suite +test.unit_tests_d.nt_BlocksInfo: BlocksInfo unit test for the MMGen Node Tools suite """ from mmgen.common import * @@ -108,7 +107,10 @@ class unit_tests: def parse_cmd_args(self,name,ut): def test(spec,foo,chk): - b = BlocksInfo(spec if type(spec) == tuple else [spec],dummyOpt(),dummyRPC()) + b = BlocksInfo( + spec if type(spec) == tuple else [spec], + dummyOpt(), + dummyRPC() ) ret = (b.first,b.last,b.block_list) vmsg('{:13} => {}'.format( (repr(spec) if type(spec) == tuple else spec), From 30d8a4a8717716a1d3ee2fad7ea74e2b8b7e4264 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 28 Mar 2023 18:16:33 +0000 Subject: [PATCH 023/108] update for MMGen v13.3.dev42 --- mmgen_node_tools/BlocksInfo.py | 38 +++++------ mmgen_node_tools/PeerBlocks.py | 9 ++- mmgen_node_tools/PollDisplay.py | 3 +- mmgen_node_tools/Ticker.py | 76 ++++++++++----------- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 17 +++-- mmgen_node_tools/main_blocks_info.py | 19 +++--- mmgen_node_tools/main_feeview.py | 44 ++++++------ mmgen_node_tools/main_halving_calculator.py | 17 +++-- mmgen_node_tools/main_netrate.py | 7 +- mmgen_node_tools/main_peerblocks.py | 13 ++-- mmgen_node_tools/main_ticker.py | 9 ++- mmgen_node_tools/main_txfind.py | 15 ++-- setup.cfg | 2 +- test/test_py_d/ts_main.py | 2 +- test/test_py_d/ts_misc.py | 4 +- test/test_py_d/ts_regtest.py | 18 +++-- test/unit_tests_d/ut_BlocksInfo.py | 9 ++- 18 files changed, 147 insertions(+), 157 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 27dfa08..70d9821 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -109,7 +109,7 @@ class BlocksInfo: def parse_cslist(cls,uarg,full_set,dfl_set,desc): def make_list(m,func): - groups_lc = [set(e.lower() for e in g.split(',')) for g in m.groups()] + groups_lc = [set(e.lower() for e in gi.split(',')) for gi in m.groups()] for group in groups_lc: for e in group: if e not in full_set_lc: @@ -135,7 +135,7 @@ class BlocksInfo: else: die(1,f'{uarg}: invalid parameter') - def __init__(self,cmd_args,opt,rpc): + def __init__(self,cfg,cmd_args,rpc): def parse_cs_uarg(uarg,full_set,dfl_set,desc): return ( @@ -144,10 +144,10 @@ class BlocksInfo: ) def get_fields(): - return parse_cs_uarg(opt.fields,list(self.fields),self.dfl_fields,'field') + return parse_cs_uarg(self.cfg.fields,list(self.fields),self.dfl_fields,'field') def get_stats(): - return parse_cs_uarg(opt.stats.lower(),self.all_stats,self.dfl_stats,'stat') + return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') def parse_cmd_args(): # => (block_list, first, last, step) if not cmd_args: @@ -163,8 +163,8 @@ class BlocksInfo: else: return ([self.conv_blkspec(a) for a in cmd_args],None,None,None) + self.cfg = cfg self.rpc = rpc - self.opt = opt self.tip = rpc.blockcount from_satoshi = self.rpc.proto.coin_amt.satoshi @@ -198,7 +198,7 @@ class BlocksInfo: 'di': lambda arg: '{:.2e}'.format(arg), } - if g.coin == 'BCH': + if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), 'fe': lambda arg: str(int(arg * to_satoshi)), @@ -206,26 +206,26 @@ class BlocksInfo: }) self.fnames = tuple( - [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] if opt.header_info else - get_fields() if opt.fields else + [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] if self.cfg.header_info else + get_fields() if self.cfg.fields else self.dfl_fields ) - if opt.miner_info and 'miner' not in self.fnames: + if self.cfg.miner_info and 'miner' not in self.fnames: self.fnames += ('miner',) - self.stats = get_stats() if opt.stats else self.dfl_stats + self.stats = get_stats() if self.cfg.stats else self.dfl_stats # Display diff stats by default only if user-requested range ends with chain tip - if 'diff' in self.stats and not opt.stats and self.last != self.tip: + if 'diff' in self.stats and not self.cfg.stats and self.last != self.tip: self.stats.remove('diff') - if {'avg','col_avg'} <= set(self.stats) and opt.stats_only: + if {'avg','col_avg'} <= set(self.stats) and self.cfg.stats_only: self.stats.remove('col_avg') if {'avg','mini_avg'} <= set(self.stats): self.stats.remove('mini_avg') - if opt.full_stats: + if self.cfg.full_stats: add_fnames = {fname for sname in self.stats for fname in self.stats_deps[sname]} self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames ) else: @@ -397,7 +397,7 @@ class BlocksInfo: self.t_cur = self.prev_hdrs[n]['time'] ret = await self.process_block(self.hdrs[n]) self.res.append(ret) - if self.fnames and not self.opt.stats_only: + if self.fnames and not self.cfg.stats_only: self.output_block(ret,n) def output_block(self,data,n): @@ -447,7 +447,7 @@ class BlocksInfo: return '---' else: cb = bytes.fromhex(bd['vin'][0]['coinbase']) - if self.opt.raw_miner_info: + if self.cfg.raw_miner_info: return repr(cb) else: trmap_in = { @@ -670,7 +670,7 @@ class BlocksInfo: return ( sname, (d.hdr,) + tuple(gen()) ) def process_stats_pre(self,i): - if (self.fnames and not self.opt.stats_only) or i != 0: + if (self.fnames and not self.cfg.stats_only) or i != 0: Msg('') def finalize_output(self): pass @@ -707,9 +707,9 @@ class BlocksInfo: class JSONBlocksInfo(BlocksInfo): - def __init__(self,cmd_args,opt,rpc): - super().__init__(cmd_args,opt,rpc) - if opt.json_raw: + def __init__(self,cfg,cmd_args,opt,rpc): + super().__init__(cfg,cmd_args,opt,rpc) + if self.cfg.json_raw: self.output_block = self.output_block_raw self.fmt_stat_item = self.fmt_stat_item_raw Msg_r('{') diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index 6df9894..0b386fb 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -14,7 +14,6 @@ mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes import asyncio from collections import namedtuple -from mmgen.globalvars import g from mmgen.util import msg,msg_r,is_int from mmgen.term import get_term,get_terminal_size,get_char from mmgen.ui import line_input @@ -30,15 +29,15 @@ class Display(PollDisplay): poll_secs = 2 - def __init__(self): + def __init__(self,cfg): - super().__init__() + super().__init__(cfg) global term,term_width if not term: term = get_term() term.init(noecho=True) - term_width = g.columns or get_terminal_size().width + term_width = self.cfg.columns or get_terminal_size().width msg_r(CUR_HOME+ERASE_ALL+CUR_HOME) async def get_info(self,rpc): @@ -69,7 +68,7 @@ class Display(PollDisplay): msg('') term.reset() # readline required for correct operation here; without it, user must re-type first digit - ret = line_input('peer number> ',insert_txt=s,hold_protect=False) + ret = line_input( self.cfg, 'peer number> ', insert_txt=s, hold_protect=False ) term.init(noecho=True) self.enable_display = False # prevent display from updating before process_input() return ret diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index bff4683..84a6023 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -22,7 +22,8 @@ class PollDisplay(): input = None poll_secs = 1 - def __init__(self): + def __init__(self,cfg): + self.cfg = cfg self.info_lock = threading.Lock() # self.info accessed by 2 threads self.display_kill_flag = threading.Event() diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7fe8b1d..e4b147c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -28,10 +28,8 @@ import sys,os,time,json,yaml from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.opts import opt -from mmgen.globalvars import g from mmgen.color import * -from mmgen.util import die,fmt_list,msg,msg_r,Msg,vmsg,suf,fmt,stdout_or_pager +from mmgen.util import die,fmt_list,msg,msg_r,suf,fmt homedir = os.getenv('HOME') cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') @@ -173,23 +171,23 @@ def get_src_data(curl_cmd): if cfg.btc_only: fn = os.path.join(cfg.cachedir,'ticker-btc.json') - timeout = 5 if g.test_suite else btc_ratelimit + timeout = 5 if gcfg.test_suite else btc_ratelimit else: fn = os.path.join(cfg.cachedir,'ticker.json') - timeout = 5 if g.test_suite else ratelimit + timeout = 5 if gcfg.test_suite else ratelimit fn_rel = os.path.relpath(fn,start=homedir) if not os.path.exists(fn): open(fn,'w').write('{}') - if opt.cached_data: + if gcfg.cached_data: json_text = open(fn).read() else: elapsed = int(time.time() - os.stat(fn).st_mtime) if elapsed >= timeout: msg_r(f'Fetching data from {api_host}...') - vmsg('') + gcfg._util.vmsg('') try: cp = run(curl_cmd,check=True,stdout=PIPE) except CalledProcessError as e: @@ -212,14 +210,14 @@ def get_src_data(curl_cmd): die(2,'Retrieved data is not valid JSON, exiting') if not data: - if opt.cached_data: + if gcfg.cached_data: die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') else: die(2,'Remote host returned no data!') elif 'error' in data: die(1,data['error']) - if opt.cached_data: + if gcfg.cached_data: msg(f'Using cached data from ~/{fn_rel}') else: open(fn,'w').write(json_text) @@ -248,7 +246,7 @@ def main(cfg_parm,cfg_in_parm): '--header', 'Accept: application/json', ] + (['--proxy', cfg.proxy] if cfg.proxy else []) + - (['--silent'] if not opt.verbose else []) + + (['--silent'] if not gcfg.verbose else []) + [api_url + ('/btc-bitcoin' if cfg.btc_only else '')] ) @@ -264,27 +262,27 @@ def main(cfg_parm,cfg_in_parm): update_sample_file(cfg_in.cfg_file) update_sample_file(cfg_in.portfolio_file) - if opt.portfolio and not cfg_in.portfolio: + if gcfg.portfolio and not cfg_in.portfolio: die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( os.path.relpath(cfg_in.portfolio_file,start=homedir))) curl_cmd = get_curl_cmd() - if opt.print_curl: + if gcfg.print_curl: Msg(curl_cmd + '\n' + ' '.join(curl_cmd)) return parsed_json = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) - if opt.list_ids: + if gcfg.list_ids: from mmgen.ui import do_pager do_pager('\n'.join(e['id'] for e in parsed_json)) return global now - now = 1659465400 if g.test_suite else time.time() # 1659524400 1659445900 + now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 - stdout_or_pager( + gcfg._util.stdout_or_pager( '\n'.join(getattr(Ticker,cfg.clsname)(dict(gen_data(parsed_json))).gen_output()) + '\n' ) @@ -366,12 +364,12 @@ def make_cfg(cmd_args,cfg_in): usr_columns ) def get_portfolio_assets(ret=()): - if cfg_in.portfolio and opt.portfolio: + if cfg_in.portfolio and gcfg.portfolio: ret = (parse_asset_id(e,True) for e in cfg_in.portfolio) - return ( 'portfolio', tuple(e for e in ret if (not opt.btc) or e.symbol == 'BTC') ) + return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') ) def get_portfolio(): - return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not opt.btc) or k == 'btc-bitcoin'} + return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'} def parse_add_precision(s): if not s: @@ -385,8 +383,8 @@ def make_cfg(cmd_args,cfg_in): def create_rows(): rows = ( ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin',parse_asset_id('btc-bitcoin')) if opt.btc else - get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if opt.add_columns else None ) + ('bitcoin',parse_asset_id('btc-bitcoin')) if gcfg.btc else + get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None ) ) for hdr,data in ( @@ -416,8 +414,8 @@ def make_cfg(cmd_args,cfg_in): asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset']) asset_tuple = namedtuple('asset_tuple',['symbol','id']) - usr_rows = parse_usr_asset_arg(opt.add_rows) - usr_columns = parse_usr_asset_arg(opt.add_columns) + usr_rows = parse_usr_asset_arg(gcfg.add_rows) + usr_columns = parse_usr_asset_arg(gcfg.add_columns) query = parse_query_arg(cmd_args[0]) if cmd_args else None return cfg_tuple( @@ -425,19 +423,19 @@ def make_cfg(cmd_args,cfg_in): usr_rows = usr_rows, usr_columns = usr_columns, query = query, - adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(opt.adjust or 0) ), + adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ), clsname = 'trading' if query else 'overview', - btc_only = opt.btc, - add_prec = parse_add_precision(opt.add_precision), - cachedir = opt.cachedir or cfg_in.cfg.get('cachedir') or cachedir, - proxy = None if opt.proxy == '' else (opt.proxy or cfg_in.cfg.get('proxy')), - portfolio = get_portfolio() if cfg_in.portfolio and opt.portfolio and not query else None + btc_only = gcfg.btc, + add_prec = parse_add_precision(gcfg.add_precision), + cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or cachedir, + proxy = None if gcfg.proxy == '' else (gcfg.proxy or cfg_in.cfg.get('proxy')), + portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None ) def get_cfg_in(): ret = namedtuple('cfg_in_data',['cfg','portfolio','cfg_file','portfolio_file']) cfg_file,portfolio_file = ( - [os.path.join(g.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)] + [os.path.join(gcfg.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)] ) cfg_data,portfolio_data = ( [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None for fn in (cfg_file,portfolio_file)] @@ -466,10 +464,10 @@ class Ticker: def __init__(self,data): - self.comma = ',' if opt.thousands_comma else '' + self.comma = ',' if gcfg.thousands_comma else '' self.col1_wid = max(len('TOTAL'),( - max(len(self.create_label(d['id'])) for d in data.values()) if opt.name_labels else + max(len(self.create_label(d['id'])) for d in data.values()) if gcfg.name_labels else max(len(d['symbol']) for d in data.values()) )) + 1 @@ -482,7 +480,7 @@ class Ticker: def format_last_update_col(self,cross_assets=()): - if opt.elapsed: + if gcfg.elapsed: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: @@ -627,7 +625,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( - lbl = (self.create_label(d['id']) if opt.name_labels else d['symbol']), + lbl = (self.create_label(d['id']) if gcfg.name_labels else d['symbol']), pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), upd = d.get('last_updated_fmt'), @@ -670,13 +668,13 @@ class Ticker: [asset.id for asset in self.usr_col_assets] + [a for a,b in ( ( 'btc-bitcoin', not cfg.btc_only ), - ( 'pct7d', opt.percent_change ), - ( 'pct24h', opt.percent_change ), - ( 'update_time', opt.update_time ), + ( 'pct7d', gcfg.percent_change ), + ( 'pct24h', gcfg.percent_change ), + ( 'update_time', gcfg.update_time ), ) if b] ) cols2 = list(cols) - if opt.update_time: + if gcfg.update_time: cols2.pop() cols2.append('amt') @@ -759,7 +757,7 @@ class Ticker: if self.show_adj: self.fs_str += ' {p_adj}' self.hl_wid += self.max_wid + 1 - if opt.update_time: + if gcfg.update_time: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 @@ -772,7 +770,7 @@ class Ticker: if self.show_adj else '' ) return self.fs_str.format( - lbl = (self.create_label(id) if opt.name_labels else d['symbol']), + lbl = (self.create_label(id) if gcfg.name_labels else d['symbol']), p_spot = green(p_spot) if id in self.hl_ids else p_spot, p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, upd = d.get('last_updated_fmt'), diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1bc37c8..4348263 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev14 +3.1.dev15 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index b0ab4f8..9f2e11e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -71,7 +71,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): fs = ( ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w,max(5,fb_w),max(4,lb_w)) - if opt.first_block else + if cfg.first_block else ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w,max(4,lb_w)) ) Msg('\n' + fs.format( @@ -109,20 +109,19 @@ def do_output_tabular(proto,addr_data,blk_hdrs): async def main(req_addrs): - from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts(need_amt=True) + proto = cfg._proto from mmgen.addr import CoinAddr addrs = [CoinAddr(proto,addr) for addr in req_addrs] from mmgen.rpc import rpc_init - rpc = await rpc_init(proto) + rpc = await rpc_init( cfg, proto ) height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') from mmgen.proto.btc.misc import scantxoutset - res = await scantxoutset( rpc, [f'addr({addr})' for addr in addrs] ) + res = await scantxoutset( cfg, rpc, [f'addr({addr})' for addr in addrs] ) if not res['success']: die(1,'UTXO scanning failed or was interrupted') @@ -155,16 +154,16 @@ async def main(req_addrs): blk_hashes = await rpc.batch_call('getblockhash', [(h,) for h in blk_heights]) blk_hdrs = await rpc.batch_call('getblockheader', [(H,) for H in blk_hashes]) - (do_output_tabular if opt.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) + (do_output_tabular if cfg.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) -cmd_args = opts.init(opts_data,init_opts={'rpc_backend':'aiohttp'}) +cfg = opts.init(opts_data,init_opts={'rpc_backend':'aiohttp'}) -if len(cmd_args) < 1: +if len(cfg._args) < 1: die(1,'This command requires at least one coin address argument') from mmgen.obj import CoinTxID,Int try: - async_run(main(cmd_args)) + async_run(main(cfg._args)) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index eb12a5c..38914c5 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -20,7 +20,9 @@ mmnode-blocks-info: Display information about a block or range of blocks """ -from mmgen.common import * +import mmgen.opts as opts +from mmgen.globalvars import gc +from mmgen.util import async_run,fmt_list from .BlocksInfo import BlocksInfo,JSONBlocksInfo opts_data = { @@ -146,29 +148,26 @@ EXAMPLES: This program requires a txindex-enabled daemon for correct operation. """ }, 'code': { - 'notes': lambda proto,s: s.format( + 'notes': lambda cfg,proto,s: s.format( I = proto.diff_adjust_interval, F = fmt_list(BlocksInfo.fields,fmt='bare'), S = fmt_list(BlocksInfo.all_stats,fmt='bare'), - p = g.prog_name, + p = gc.prog_name, ) } } -cmd_args = opts.init(opts_data) +cfg = opts.init(opts_data) async def main(): - from mmgen.protocol import init_proto_from_opts from mmgen.rpc import rpc_init - proto = init_proto_from_opts(need_amt=True) + cls = JSONBlocksInfo if cfg.json else BlocksInfo - cls = JSONBlocksInfo if opt.json else BlocksInfo + m = cls( cfg, cfg._args, await rpc_init(cfg,cfg._proto) ) - m = cls( cmd_args, opt, await rpc_init(proto) ) - - if m.fnames and not opt.no_header: + if m.fnames and not cfg.no_header: m.print_header() await m.process_blocks() diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index ce1fbdb..f5212e3 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -39,7 +39,7 @@ fee_brackets = [ 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 2100000000000000, ] -opts.init({ +cfg = opts.init({ 'sets': [ ('detail',True,'ranges',True), ('detail',True,'show_mb_col',True), @@ -78,17 +78,17 @@ factors. } }) -if opt.ignore_below: - if opt.show_empty: +if cfg.ignore_below: + if cfg.show_empty: die(1,'Conflicting options: --ignore-below, --show-empty') - ignore_below = parse_bytespec(opt.ignore_below) + ignore_below = parse_bytespec(cfg.ignore_below) precision = ( - check_int_between(opt.precision,min_prec,max_prec,'--precision arg') - if opt.precision else dfl_prec ) + check_int_between(cfg.precision,min_prec,max_prec,'--precision arg') + if cfg.precision else dfl_prec ) from mmgen.term import get_terminal_size -width = g.columns or get_terminal_size().width +width = cfg.columns or get_terminal_size().width class fee_bracket: def __init__(self,top,bottom): @@ -103,6 +103,7 @@ def log(data,fn): from mmgen.rpc import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( + cfg = cfg, outfile = fn, data = json.dumps(data,cls=json_encoder,sort_keys=True,indent=4), desc = 'mempool', @@ -130,9 +131,9 @@ def create_data(coin_amt,mempool): # calculate cumulative byte totals, filter rows: tBytes = 0 for i in out: - if not (i.tx_bytes or opt.show_empty): + if not (i.tx_bytes or cfg.show_empty): i.skip = True - if opt.ignore_below and i.tx_bytes < ignore_below: + if cfg.ignore_below and i.tx_bytes < ignore_below: i.skip = True i.tx_bytes_cum = tBytes tBytes += i.tx_bytes @@ -150,15 +151,15 @@ def gen_header(host,mempool,blockcount): TX count: {len(mempool)} """)).strip() - if opt.show_empty: + if cfg.show_empty: yield('Displaying all fee brackets') - elif opt.ignore_below: + elif cfg.ignore_below: yield('Ignoring fee brackets with less than {:,} bytes ({})'.format( ignore_below, int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), )) - if opt.include_current: + if cfg.include_current: yield('Including transactions in current fee bracket in Total MB amounts') def fmt_mb(n): @@ -168,11 +169,11 @@ def gen_body(data): tx_bytes_max = max((i.tx_bytes for i in data),default=0) top_max = max((i.top for i in data),default=0) bot_max = max((i.bottom for i in data),default=0) - col1_w = max(len(f'{bot_max}-{top_max}') if opt.ranges else len(f'{top_max}'),6) - col2_w = len(fmt_mb(tx_bytes_max)) if opt.show_mb_col else 0 + col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'),6) + col2_w = len(fmt_mb(tx_bytes_max)) if cfg.show_mb_col else 0 col3_w = len(fmt_mb(data[-1].tx_bytes_cum)) if data else 0 col4_w = width - col1_w - col2_w - col3_w - (4 if col2_w else 3) - if opt.show_mb_col: + if cfg.show_mb_col: fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w,col2_w,col3_w) else: fs = '{a:<%i} {c:>%i} {d}' % (col1_w,col3_w) @@ -182,9 +183,9 @@ def gen_body(data): for i in data: if not i.skip: - cum_bytes = i.tx_bytes_cum + i.tx_bytes if opt.include_current else i.tx_bytes_cum + cum_bytes = i.tx_bytes_cum + i.tx_bytes if cfg.include_current else i.tx_bytes_cum yield(fs.format( - a = '{}-{}'.format(i.bottom,i.top) if opt.ranges else i.top, + a = '{}-{}'.format(i.bottom,i.top) if cfg.ranges else i.top, b = fmt_mb(i.tx_bytes), c = fmt_mb(cum_bytes), d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) )) @@ -197,20 +198,19 @@ def gen_body(data): async def main(): - from mmgen.protocol import init_proto_from_opts global proto - proto = init_proto_from_opts(need_amt=True) + proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init(proto) + c = await rpc_init( cfg, proto ) mempool = await c.call('getrawmempool',True) - if opt.log: + if cfg.log: log(mempool,'mempool.json') data = create_data(proto.coin_amt,mempool) - stdout_or_pager( + cfg._util.stdout_or_pager( '\n'.join(gen_header( c.host, mempool, diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 5cbf2b8..e92d161 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -26,7 +26,7 @@ from mmgen.common import * bdr_proj = 9.95 -opts.init({ +cfg = opts.init({ 'sets': [('mined',True,'list',True)], 'text': { 'desc': 'Estimate date(s) of future block subsidy halving(s)', @@ -43,8 +43,8 @@ opts.init({ """ } }) -if opt.bdr_proj: - bdr_proj = float(opt.bdr_proj) +if cfg.bdr_proj: + bdr_proj = float(cfg.bdr_proj) def date(t): return '{}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*time.gmtime(t)[:6]) @@ -61,16 +61,15 @@ def time_diff_warning(t_diff): async def main(): - from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts(need_amt=True) + proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init(proto) + c = await rpc_init( cfg, proto ) tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' remaining = proto.halving_interval - tip % proto.halving_interval - sample_size = int(opt.sample_size) if opt.sample_size else min(tip-1,max(remaining,144)) + sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1,max(remaining,144)) cur,old = await c.gathered_call('getblockstats',((tip,),(tip - sample_size,))) @@ -134,7 +133,7 @@ async def main(): fs = ( ' {a:<7} {b:>8} {c:19}{d:2} {e:10} {f}', ' {a:<7} {b:>8} {c:19}{d:2} {e:10} {f:17} {g:17} {h}' - )[bool(opt.mined)] + )[bool(cfg.mined)] print( f'Historical/Estimated/Projected Halvings ({proto.coin}):\n\n' @@ -174,7 +173,7 @@ async def main(): ) for n,sub,blk,mined,total_mined,bdr,t in gen_data()) ) - if opt.list: + if cfg.list: await print_halvings() else: print_current_stats() diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index fd86983..78c9823 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -34,17 +34,14 @@ opts_data = { } } -cmd_args = opts.init(opts_data) +cfg = opts.init(opts_data) ERASE_LINE,CUR_UP = '\033[K','\033[1A' async def main(): - from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts() - from mmgen.rpc import rpc_init - c = await rpc_init(proto) + c = await rpc_init( cfg, cfg._proto ) async def get_data(): d = await c.call('getnettotals') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 9e6a0ec..af723c9 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -20,7 +20,7 @@ mmnode-peerblocks: List blocks in flight, disconnect stalling nodes """ -from mmgen.opts import init +import mmgen.opts as opts from mmgen.util import async_run opts_data = { @@ -36,17 +36,14 @@ opts_data = { async def main(): - init(opts_data) - - from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts() + cfg = opts.init(opts_data) from mmgen.rpc import rpc_init - rpc = await rpc_init(proto) + rpc = await rpc_init( cfg, cfg._proto ) from .PeerBlocks import BlocksDisplay,PeersDisplay - blocks = BlocksDisplay() - peers = PeersDisplay() + blocks = BlocksDisplay(cfg) + peers = PeersDisplay(cfg) while True: await blocks.run(rpc) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index a25e884..da96bb8 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -197,12 +197,15 @@ To add a portfolio, edit the file } } -cmd_args = opts.init(opts_data,do_post_init=True) +gcfg = opts.init(opts_data,do_post_init=True) + +import mmgen_node_tools.Ticker as Ticker +Ticker.gcfg = gcfg cfg_in = get_cfg_in() -cfg = make_cfg(cmd_args,cfg_in) +cfg = make_cfg(gcfg._args,cfg_in) -opts.post_init() +opts.post_init(gcfg) main(cfg,cfg_in) diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index d0a04e2..4c0bcc2 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -58,14 +58,11 @@ async def main(txid): if len(txid) != 64 or not is_hex_str(txid): die(2,f'{txid}: invalid transaction ID') - if opt.verbose: + if cfg.verbose: msg(f'TxID: {txid}') - from mmgen.protocol import init_proto_from_opts - proto = init_proto_from_opts() - from mmgen.rpc import rpc_init - c = await rpc_init(proto) + c = await rpc_init(cfg._proto) exitval = 0 try: @@ -85,11 +82,11 @@ async def main(txid): return exitval -cmd_args = opts.init(opts_data) +cfg = opts.init(opts_data) -msgs = msg_data['quiet' if opt.quiet else 'normal'] +msgs = msg_data['quiet' if cfg.quiet else 'normal'] -if len(cmd_args) != 1: +if len(cfg._args) != 1: die(1,'One transaction ID must be specified') -sys.exit(async_run(main(cmd_args[0]))) +sys.exit(async_run(main(cfg._args[0]))) diff --git a/setup.cfg b/setup.cfg index 65b9639..e669735 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev20 + mmgen>=13.3.dev42 packages = mmgen_node_tools diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index 171ffc5..162e3fb 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -45,7 +45,7 @@ class TestSuiteMain(TestSuiteBase): f'mmnode-peerblocks', args, pexpect_spawn = pexpect_spawn ) - if opt.exact_output: # disable echoing of input + if cfg.exact_output: # disable echoing of input t.p.logfile = None t.p.logfile_read = sys.stdout if expect_list: diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index cc4c22d..c4536d6 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -97,7 +97,7 @@ class TestSuiteScripts(TestSuiteBase): @property def nt_datadir(self): - return os.path.join( g.data_dir_root, 'node_tools' ) + return os.path.join( cfg.data_dir_root, 'node_tools' ) def ticker_setup(self): self.spawn('',msg_only=True) @@ -120,7 +120,7 @@ class TestSuiteScripts(TestSuiteBase): def ticker2(self): t = self.ticker(cached=False) - if not opt.skipping_deps: + if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') t.expect('proxy host could not be resolved') diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index fd19099..e208373 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -13,8 +13,6 @@ test.test_py_d.ts_regtest: Regtest tests for the test.py test suite """ import os -from mmgen.globalvars import g -from mmgen.opts import opt from mmgen.util import die,gmsg from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest @@ -28,7 +26,7 @@ args2 = ['--bob','--rpc-backend=http'] def gen_addrs(proto,network,keys): from mmgen.tool.api import tool_api - tool = tool_api() + tool = tool_api(cfg) tool.init_coin(proto.coin,'regtest') tool.addrtype = proto.mmtypes[-1] return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] @@ -105,9 +103,9 @@ class TestSuiteRegtest(TestSuiteBase): return if self.proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') - self.proto = init_proto(self.proto.coin,network='regtest',need_amt=True) + self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) - self.regtest = MMGenRegtest(self.proto.coin) + self.regtest = MMGenRegtest(cfg,self.proto.coin) def setup(self): stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) @@ -128,7 +126,7 @@ class TestSuiteRegtest(TestSuiteBase): return self.halving_calculator(['--help'],['USAGE:']) def halving_calculator2(self): - return self.halving_calculator([],['Current block: 393',f'Current block subsidy: 12.5 {g.coin}']) + return self.halving_calculator([],['Current block: 393',f'Current block subsidy: 12.5 {cfg.coin}']) def halving_calculator3(self): return self.halving_calculator(['--list'],['33 4950','0']) @@ -158,7 +156,7 @@ class TestSuiteRegtest(TestSuiteBase): return self.addrbal( args2 + [self.addrs[0]], [ - f'Balance: 0.357 {g.coin}', + f'Balance: 0.357 {cfg.coin}', '2 unspent outputs in 2 blocks', '394','0.123', '395','0.234' @@ -246,7 +244,7 @@ class TestSuiteRegtest(TestSuiteBase): ]) def blocks_info4(self): - n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if g.coin == 'BCH' else (2,1,4,6,3,12) + n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) return self.blocks_info( args1 + ['--miner-info','--fields=all','--stats=all','+3'], [ 'Averages', f'nTx: {n1}', @@ -267,7 +265,7 @@ class TestSuiteRegtest(TestSuiteBase): from mmgen.tool.api import tool_api from collections import namedtuple - t = tool_api() + t = tool_api(cfg) t.init_coin(self.proto.coin,self.proto.network) t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32' wp = namedtuple('wifaddrpair',['wif','addr']) @@ -384,7 +382,7 @@ class TestSuiteRegtest(TestSuiteBase): return self._feeview([]) def stop(self): - if opt.no_daemon_stop: + if cfg.no_daemon_stop: self.spawn('',msg_only=True) msg_r('(leaving daemon running by user request)') return 'ok' diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/unit_tests_d/ut_BlocksInfo.py index ab09882..e65e314 100755 --- a/test/unit_tests_d/ut_BlocksInfo.py +++ b/test/unit_tests_d/ut_BlocksInfo.py @@ -7,6 +7,8 @@ from mmgen.common import * from mmgen.exception import * from mmgen_node_tools.BlocksInfo import BlocksInfo +from ..include.common import cfg,vmsg + tip = 50000 range_vecs = ( # First Last FromTip nBlocks Step First Last BlockList @@ -70,12 +72,13 @@ class dummyRPC: class coin_amt: satoshi = 0.00000001 -class dummyOpt: +class dummyCfg: fields = None stats = None miner_info = None header_info = None full_stats = None + coin = 'BTC' class unit_tests: @@ -91,7 +94,7 @@ class unit_tests: def parse_rangespec(self,name,ut): - b = BlocksInfo(0,dummyOpt(),dummyRPC()) + b = BlocksInfo(dummyCfg(),None,dummyRPC()) def test(spec,chk,foo): ret = b.parse_rangespec(spec) @@ -108,8 +111,8 @@ class unit_tests: def test(spec,foo,chk): b = BlocksInfo( + dummyCfg(), spec if type(spec) == tuple else [spec], - dummyOpt(), dummyRPC() ) ret = (b.first,b.last,b.block_list) vmsg('{:13} => {}'.format( From 89bda2f00fe45b9c481dd78cdf060df1cf4fd476 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 2 Apr 2023 10:12:47 +0000 Subject: [PATCH 024/108] update for MMGen v13.3.dev43 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4348263..6496c42 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev15 +3.1.dev16 diff --git a/setup.cfg b/setup.cfg index e669735..6eb85ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev42 + mmgen>=13.3.dev43 packages = mmgen_node_tools From 435256a99affa5eb59b8d04c486610579b46bee7 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 4 Apr 2023 16:00:08 +0000 Subject: [PATCH 025/108] test.py regtest: add test group for `mmnode-netrate` --- test/test_py_d/ts_regtest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index e208373..9b72cb2 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -41,6 +41,7 @@ class TestSuiteRegtest(TestSuiteBase): deterministic = False cmd_group_in = ( ('setup', 'regtest mode setup'), + ('subgroup.netrate', []), ('subgroup.halving_calculator', []), ('subgroup.fund_addrbal', []), ('subgroup.addrbal', ['fund_addrbal']), @@ -49,6 +50,11 @@ class TestSuiteRegtest(TestSuiteBase): ('stop', 'stopping regtest daemon'), ) cmd_subgroups = { + 'netrate': ( + "'mmnode-netrate' script", + ('netrate1', "netrate (--help)"), + ('netrate2', "netrate"), + ), 'halving_calculator': ( "'mmnode-halving-calculator' script", ('halving_calculator1', "halving calculator (--help)"), @@ -117,6 +123,19 @@ class TestSuiteRegtest(TestSuiteBase): t.expect(s) return t + def netrate(self,add_args,expect_str): + t = self.spawn( 'mmnode-netrate', args1 + add_args ) + t.expect(expect_str,regex=True) + return t + + def netrate1(self): + return self.netrate( ['--help'], 'USAGE:.*' ) + + def netrate2(self): + t = self.netrate( [], 'sent:.*' ) + t.kill(2) + return t + def halving_calculator(self,add_args,expect_list): t = self.spawn('mmnode-halving-calculator',args1+add_args) t.match_expect_list(expect_list) From a1beac67e234c40a6869dac1ebb4a4d1ba9ed483 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 4 Apr 2023 16:00:29 +0000 Subject: [PATCH 026/108] update for MMGen v13.3.dev44 --- mmgen_node_tools/BlocksInfo.py | 5 +++-- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 11 ++++++----- mmgen_node_tools/main_blocks_info.py | 7 +++---- mmgen_node_tools/main_feeview.py | 11 +++++++---- mmgen_node_tools/main_halving_calculator.py | 10 +++++++--- mmgen_node_tools/main_netrate.py | 10 ++++++---- mmgen_node_tools/main_peerblocks.py | 9 ++++----- mmgen_node_tools/main_ticker.py | 6 +++--- mmgen_node_tools/main_txfind.py | 7 ++++--- setup.cfg | 2 +- 11 files changed, 45 insertions(+), 35 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 70d9821..25e1caf 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -24,7 +24,7 @@ import re,json from collections import namedtuple from time import strftime,gmtime -from mmgen.common import * +from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int from mmgen.rpc import json_encoder class BlocksInfo: @@ -362,7 +362,8 @@ class BlocksInfo: first = self.conv_blkspec(first) last = self.conv_blkspec(last or first) - if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + if p.debug: + msg(repr(self.range_data(first,last,from_tip,nblocks,step))) if first > last: die(1,f'{first}-{last}: invalid block range') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6496c42..623245b 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev16 +3.1.dev17 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 9f2e11e..177bac4 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -12,7 +12,10 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain """ -from mmgen.common import * +from mmgen.obj import CoinTxID,Int +from mmgen.cfg import Config +from mmgen.util import msg,Msg,die,suf,make_timestr,async_run +from mmgen.color import red opts_data = { 'text': { @@ -115,7 +118,7 @@ async def main(req_addrs): addrs = [CoinAddr(proto,addr) for addr in req_addrs] from mmgen.rpc import rpc_init - rpc = await rpc_init( cfg, proto ) + rpc = await rpc_init(cfg) height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') @@ -156,13 +159,11 @@ async def main(req_addrs): (do_output_tabular if cfg.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) -cfg = opts.init(opts_data,init_opts={'rpc_backend':'aiohttp'}) +cfg = Config( opts_data=opts_data, init_opts={'rpc_backend':'aiohttp'} ) if len(cfg._args) < 1: die(1,'This command requires at least one coin address argument') -from mmgen.obj import CoinTxID,Int - try: async_run(main(cfg._args)) except KeyboardInterrupt: diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 38914c5..1fe5e82 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -20,8 +20,7 @@ mmnode-blocks-info: Display information about a block or range of blocks """ -import mmgen.opts as opts -from mmgen.globalvars import gc +from mmgen.cfg import gc,Config from mmgen.util import async_run,fmt_list from .BlocksInfo import BlocksInfo,JSONBlocksInfo @@ -157,7 +156,7 @@ This program requires a txindex-enabled daemon for correct operation. } } -cfg = opts.init(opts_data) +cfg = Config(opts_data=opts_data) async def main(): @@ -165,7 +164,7 @@ async def main(): cls = JSONBlocksInfo if cfg.json else BlocksInfo - m = cls( cfg, cfg._args, await rpc_init(cfg,cfg._proto) ) + m = cls( cfg, cfg._args, await rpc_init(cfg) ) if m.fnames and not cfg.no_header: m.print_header() diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index f5212e3..527dc85 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -20,7 +20,8 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool """ -from mmgen.common import * +from mmgen.cfg import Config +from mmgen.util import async_run,die,fmt,make_timestr,check_int_between from mmgen.util2 import int2bytespec,parse_bytespec min_prec,max_prec,dfl_prec = (0,6,4) @@ -39,7 +40,7 @@ fee_brackets = [ 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 2100000000000000, ] -cfg = opts.init({ +opts_data = { 'sets': [ ('detail',True,'ranges',True), ('detail',True,'show_mb_col',True), @@ -75,8 +76,10 @@ Note that there is no global mempool in Bitcoin, and your node’s mempool may differ significantly from those of mining nodes depending on uptime and other factors. """ + } } -}) + +cfg = Config(opts_data=opts_data) if cfg.ignore_below: if cfg.show_empty: @@ -202,7 +205,7 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init( cfg, proto ) + c = await rpc_init(cfg) mempool = await c.call('getrawmempool',True) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index e92d161..1e35b2a 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -22,11 +22,13 @@ mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) import time from decimal import Decimal -from mmgen.common import * + +from mmgen.cfg import Config +from mmgen.util import async_run bdr_proj = 9.95 -cfg = opts.init({ +opts_data = { 'sets': [('mined',True,'list',True)], 'text': { 'desc': 'Estimate date(s) of future block subsidy halving(s)', @@ -41,7 +43,9 @@ cfg = opts.init({ -s, --sample-size=N Block range to calculate block discovery interval for next halving estimate (default: dynamically calculated) """ } -}) +} + +cfg = Config(opts_data=opts_data) if cfg.bdr_proj: bdr_proj = float(cfg.bdr_proj) diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index 78c9823..da0b4dc 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -20,8 +20,10 @@ mmnode-netrate: Bitcoin daemon network rate monitor """ -import time -from mmgen.common import * +import sys,time + +from mmgen.cfg import Config +from mmgen.util import async_run opts_data = { 'text': { @@ -34,14 +36,14 @@ opts_data = { } } -cfg = opts.init(opts_data) +cfg = Config(opts_data=opts_data) ERASE_LINE,CUR_UP = '\033[K','\033[1A' async def main(): from mmgen.rpc import rpc_init - c = await rpc_init( cfg, cfg._proto ) + c = await rpc_init(cfg) async def get_data(): d = await c.call('getnettotals') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index af723c9..56b412c 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -20,9 +20,6 @@ mmnode-peerblocks: List blocks in flight, disconnect stalling nodes """ -import mmgen.opts as opts -from mmgen.util import async_run - opts_data = { 'text': { 'desc': 'List blocks in flight, disconnect stalling nodes', @@ -36,10 +33,11 @@ opts_data = { async def main(): - cfg = opts.init(opts_data) + from mmgen.cfg import Config + cfg = Config(opts_data=opts_data) from mmgen.rpc import rpc_init - rpc = await rpc_init( cfg, cfg._proto ) + rpc = await rpc_init(cfg) from .PeerBlocks import BlocksDisplay,PeersDisplay blocks = BlocksDisplay(cfg) @@ -49,4 +47,5 @@ async def main(): await blocks.run(rpc) await peers.run(rpc) +from mmgen.util import async_run async_run(main()) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index da96bb8..83d0b1c 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -13,7 +13,6 @@ mmnode-ticker: Display price information for cryptocurrency and other assets """ import sys,os -from mmgen.common import * from .Ticker import * opts_data = { @@ -197,7 +196,8 @@ To add a portfolio, edit the file } } -gcfg = opts.init(opts_data,do_post_init=True) +from mmgen.cfg import Config +gcfg = Config( opts_data=opts_data, do_post_init=True ) import mmgen_node_tools.Ticker as Ticker Ticker.gcfg = gcfg @@ -206,6 +206,6 @@ cfg_in = get_cfg_in() cfg = make_cfg(gcfg._args,cfg_in) -opts.post_init(gcfg) +gcfg._post_init() main(cfg,cfg_in) diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 4c0bcc2..cabed6f 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -20,7 +20,8 @@ mmnode-txfind: Find a transaction in the blockchain or mempool """ -from mmgen.common import * +from mmgen.cfg import Config +from mmgen.util import msg,Msg,die opts_data = { 'text': { @@ -62,7 +63,7 @@ async def main(txid): msg(f'TxID: {txid}') from mmgen.rpc import rpc_init - c = await rpc_init(cfg._proto) + c = await rpc_init(cfg) exitval = 0 try: @@ -82,7 +83,7 @@ async def main(txid): return exitval -cfg = opts.init(opts_data) +cfg = Config(opts_data=opts_data) msgs = msg_data['quiet' if cfg.quiet else 'normal'] diff --git a/setup.cfg b/setup.cfg index 6eb85ec..0cdbed1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev43 + mmgen>=13.3.dev44 packages = mmgen_node_tools From c373ed7a2db983442b8b9f20b19a7754e05c280c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 1 Jul 2023 16:47:49 +0000 Subject: [PATCH 027/108] rpc_init(): skip wallet initialization --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_blocks_info.py | 2 +- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 2 +- mmgen_node_tools/main_netrate.py | 2 +- mmgen_node_tools/main_peerblocks.py | 2 +- mmgen_node_tools/main_txfind.py | 2 +- test/test_py_d/ts_regtest.py | 5 +++-- 9 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 623245b..b473ce1 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.1.dev17 +3.2.dev0 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 177bac4..a461e9e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -118,7 +118,7 @@ async def main(req_addrs): addrs = [CoinAddr(proto,addr) for addr in req_addrs] from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg) + rpc = await rpc_init(cfg,ignore_wallet=True) height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 1fe5e82..c2bbf59 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -164,7 +164,7 @@ async def main(): cls = JSONBlocksInfo if cfg.json else BlocksInfo - m = cls( cfg, cfg._args, await rpc_init(cfg) ) + m = cls( cfg, cfg._args, await rpc_init(cfg,ignore_wallet=True) ) if m.fnames and not cfg.no_header: m.print_header() diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 527dc85..72692fc 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -205,7 +205,7 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init(cfg) + c = await rpc_init(cfg,ignore_wallet=True) mempool = await c.call('getrawmempool',True) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 1e35b2a..1648d76 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -68,7 +68,7 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init( cfg, proto ) + c = await rpc_init( cfg, proto, ignore_wallet=True ) tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index da0b4dc..f1661c5 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -43,7 +43,7 @@ ERASE_LINE,CUR_UP = '\033[K','\033[1A' async def main(): from mmgen.rpc import rpc_init - c = await rpc_init(cfg) + c = await rpc_init(cfg,ignore_wallet=True) async def get_data(): d = await c.call('getnettotals') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 56b412c..6ebb5a3 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -37,7 +37,7 @@ async def main(): cfg = Config(opts_data=opts_data) from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg) + rpc = await rpc_init(cfg,ignore_wallet=True) from .PeerBlocks import BlocksDisplay,PeersDisplay blocks = BlocksDisplay(cfg) diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index cabed6f..cc8df14 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -63,7 +63,7 @@ async def main(txid): msg(f'TxID: {txid}') from mmgen.rpc import rpc_init - c = await rpc_init(cfg) + c = await rpc_init(cfg,ignore_wallet=True) exitval = 0 try: diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 9b72cb2..f8859b8 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -132,8 +132,9 @@ class TestSuiteRegtest(TestSuiteBase): return self.netrate( ['--help'], 'USAGE:.*' ) def netrate2(self): - t = self.netrate( [], 'sent:.*' ) - t.kill(2) + t = self.netrate( [], r'sent:.*' ) + t.kill(15) + t.req_exit_val = -15 return t def halving_calculator(self,add_args,expect_list): From 6074a0e42a4b2b88e07ed47584553eb4c7b76125 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 25 Sep 2023 15:53:02 +0000 Subject: [PATCH 028/108] mmnode-ticker: fixes and cleanups throughout --- mmgen_node_tools/Ticker.py | 120 ++++++++++++++++---------- mmgen_node_tools/data/ticker-cfg.yaml | 13 +-- mmgen_node_tools/main_ticker.py | 28 +++--- test/test_py_d/ts_misc.py | 8 +- 4 files changed, 102 insertions(+), 67 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index e4b147c..a370b56 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -28,11 +28,12 @@ import sys,os,time,json,yaml from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple + from mmgen.color import * -from mmgen.util import die,fmt_list,msg,msg_r,suf,fmt +from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list homedir = os.getenv('HOME') -cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') +dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' @@ -78,7 +79,7 @@ def gen_data(data): die(1,'Missing data, exiting') rows_want = { - 'id': {r.id for r in cfg.rows if getattr(r,'id',None)} - {'usd-us-dollar'}, + 'id': {r.id for r in cfg.rows if isinstance(r,tuple) and r.id} - {'usd-us-dollar'}, 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, } usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) @@ -100,25 +101,29 @@ def gen_data(data): found = { 'id': set(), 'symbol': set() } rate_assets = {} - for k in ['id','symbol']: - wants = rows_want[k] | usr_wants[k] - if wants: - for d in data: - if d[k] in wants: - if d[k] in found[k]: - die(1,dup_sym_errmsg(d[k])) - yield (d['id'],d) - found[k].add(d[k]) - if d[k] in usr_rate_assets_want[k]: - rate_assets[d['symbol']] = d # NB: using symbol instead of ID - if k == 'id' and len(found[k]) == len(wants): - break + wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} for d in data: if d['id'] == 'btc-bitcoin': btcusd = Decimal(d['price_usd']) break + for k in ('id','symbol'): + for d in data: + if wants[k]: + if d[k] in wants[k]: + if d[k] in found[k]: + die(1,dup_sym_errmsg(d[k])) + yield (d['id'],d) + found[k].add(d[k]) + wants[k].remove(d[k]) + if d[k] in usr_rate_assets_want[k]: + rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key + else: + break + + check_assets_found(usr_wants,found) + for asset in (cfg.usr_rows + cfg.usr_columns): if asset.rate: """ @@ -129,21 +134,21 @@ def gen_data(data): yield ( _id, { 'symbol': asset.symbol, 'id': _id, + 'name': ' '.join(_id.split('-')[1:]), 'price_usd': str(Decimal(ra_rate/asset.rate)), 'price_btc': str(Decimal(ra_rate/asset.rate/btcusd)), - 'last_updated': int(now), + 'last_updated': None, }) yield ('usd-us-dollar', { 'symbol': 'USD', 'id': 'usd-us-dollar', + 'name': 'US Dollar', 'price_usd': '1.0', 'price_btc': str(Decimal(1/btcusd)), - 'last_updated': int(now), + 'last_updated': None, }) - check_assets_found(usr_wants,found) - def get_src_data(curl_cmd): tor_captcha_msg = f""" @@ -166,8 +171,8 @@ def get_src_data(curl_cmd): ('' if cfg.btc_only else ', or use --cached-data or --btc') ) - if not os.path.exists(cachedir): - os.makedirs(cachedir) + if not os.path.exists(cfg.cachedir): + os.makedirs(cfg.cachedir) if cfg.btc_only: fn = os.path.join(cfg.cachedir,'ticker-btc.json') @@ -225,18 +230,18 @@ def get_src_data(curl_cmd): return data -def main(cfg_parm,cfg_in_parm): +def main(): def update_sample_file(usr_cfg_file): - src_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() + usr_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() sample_file = usr_cfg_file + '.sample' sample_data = open(sample_file).read() if os.path.exists(sample_file) else None - if src_data != sample_data: + if usr_data != sample_data: os.makedirs(os.path.dirname(sample_file),exist_ok=True) msg('{} {}'.format( ('Updating','Creating')[sample_data is None], sample_file )) - open(sample_file,'w').write(src_data) + open(sample_file,'w').write(usr_data) def get_curl_cmd(): return ([ @@ -251,8 +256,6 @@ def main(cfg_parm,cfg_in_parm): ) global cfg,cfg_in - cfg = cfg_parm - cfg_in = cfg_in_parm try: from importlib.resources import files # Python 3.9 @@ -272,21 +275,23 @@ def main(cfg_parm,cfg_in_parm): Msg(curl_cmd + '\n' + ' '.join(curl_cmd)) return - parsed_json = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) + src_data = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) if gcfg.list_ids: from mmgen.ui import do_pager - do_pager('\n'.join(e['id'] for e in parsed_json)) + do_pager('\n'.join(e['id'] for e in src_data)) return global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 + data = dict(gen_data(src_data)) + gcfg._util.stdout_or_pager( - '\n'.join(getattr(Ticker,cfg.clsname)(dict(gen_data(parsed_json))).gen_output()) + '\n' + '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n' ) -def make_cfg(cmd_args,cfg_in): +def make_cfg(): def get_rows_from_cfg(add_data=None): def gen(): @@ -295,7 +300,7 @@ def make_cfg(cmd_args,cfg_in): if add_data and k in add_data: v += tuple(add_data[k]) for e in v: - yield parse_asset_id(e,True) + yield parse_asset_id(e,require_label=True) return tuple(gen()) def parse_asset_id(s,require_label=False): @@ -304,7 +309,7 @@ def make_cfg(cmd_args,cfg_in): die(1,f'{s!r}: asset label is missing') return asset_tuple( sym.upper(), (s.lower() if label else None) ) - def parse_usr_asset_arg(s): + def parse_usr_asset_arg(key,use_cf_file=False): """ asset_id[:rate[:rate_asset]] """ @@ -324,7 +329,9 @@ def make_cfg(cmd_args,cfg_in): Decimal(rate) ), rate_asset = parse_asset_id(rate_asset) if rate_asset else None ) - return tuple(parse_parm(s2) for s2 in s.split(',')) if s else () + cl_opt = getattr(gcfg,key) + cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] + return tuple( parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt) ) def parse_query_arg(s): """ @@ -365,7 +372,7 @@ def make_cfg(cmd_args,cfg_in): def get_portfolio_assets(ret=()): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e,True) for e in cfg_in.portfolio) + ret = (parse_asset_id(e,require_label=True) for e in cfg_in.portfolio) return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') ) def get_portfolio(): @@ -410,15 +417,30 @@ def make_cfg(cmd_args,cfg_in): 'proxy', 'portfolio' ]) + global cfg_in,cfg + + cmd_args = gcfg._args + cfg_in = get_cfg_in() + query_tuple = namedtuple('query',['asset','to_asset']) asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset']) asset_tuple = namedtuple('asset_tuple',['symbol','id']) - usr_rows = parse_usr_asset_arg(gcfg.add_rows) - usr_columns = parse_usr_asset_arg(gcfg.add_columns) + usr_rows = parse_usr_asset_arg('add_rows') + usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None - return cfg_tuple( + def get_proxy(name): + proxy = getattr(gcfg,name) + return ( + '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') + else (proxy or cfg_in.cfg.get(name)) + ) + + proxy = get_proxy('proxy') + proxy = None if proxy == 'none' else proxy + + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, usr_columns = usr_columns, @@ -427,8 +449,8 @@ def make_cfg(cmd_args,cfg_in): clsname = 'trading' if query else 'overview', btc_only = gcfg.btc, add_prec = parse_add_precision(gcfg.add_precision), - cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or cachedir, - proxy = None if gcfg.proxy == '' else (gcfg.proxy or cfg_in.cfg.get('proxy')), + cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, + proxy = proxy, portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None ) @@ -489,13 +511,21 @@ class Ticker: d = self.data max_w = 0 - min_t = min( (int(d[a.id]['last_updated']) for a in cross_assets), default=None ) + + if cross_assets: + last_updated_x = [d[a.id]['last_updated'] for a in cross_assets] + min_t = min( (int(n) for n in last_updated_x if isinstance(n,int) ), default=None ) + else: + min_t = None for row in self.rows: if isinstance(row,tuple): try: - t = int(d[row.id]['last_updated']) - except KeyError: + t = int( d[row.id]['last_updated'] ) + except TypeError as e: + d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) + except KeyError as e: + msg(str(e)) pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now ) @@ -517,7 +547,7 @@ class Ticker: return d['id'] def create_label(self,id): - return ' '.join(id.split('-')[1:]).upper() + return self.data[id]['name'].upper() def gen_output(self): yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now))) diff --git a/mmgen_node_tools/data/ticker-cfg.yaml b/mmgen_node_tools/data/ticker-cfg.yaml index 2566da4..6589bfc 100644 --- a/mmgen_node_tools/data/ticker-cfg.yaml +++ b/mmgen_node_tools/data/ticker-cfg.yaml @@ -1,14 +1,15 @@ -# Indentation must be consistent! Do not mix leading tabs and spaces. +### Indentation must be consistent! Do not mix leading tabs and spaces. -# See the curl manpage for supported --proxy parameters -# For a direct connection, leave the right-hand side blank +### See the curl manpage for supported --proxy parameters +### For a direct connection, leave the right-hand side blank proxy: http://vpn-gw:8118 -# Override the default cache directory (~/.cache/mmgen-node-tools) +### Override the default cache directory (~/.cache/mmgen-node-tools): cachedir: -# Asset labels are arbitrary strings. Use as many or few as you wish. -# Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. +### Asset rows +### Asset labels are arbitrary strings. Use as many or few as you wish. +### Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. assets: coin1: - btc-bitcoin diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 83d0b1c..b8df683 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -12,7 +12,6 @@ mmnode-ticker: Display price information for cryptocurrency and other assets """ -import sys,os from .Ticker import * opts_data = { @@ -36,8 +35,8 @@ opts_data = { used to supply a USD exchange rate for missing assets. -C, --cached-data Use cached data from previous network query instead of live data from server --d, --cachedir=D Read and write cached JSON data to directory ‘D’ - instead of ‘~/{os.path.relpath(cachedir,start=homedir)}’ +-D, --cachedir=D Read and write cached JSON data to directory ‘D’ + instead of ‘~/{dfl_cachedir}’ -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -48,13 +47,14 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. +-t, --testing Print command to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column --U, --print-curl Print cURL command to standard output and exit -v, --verbose Be more verbose -w, --wide Display all optional columns (equivalent to -punT) -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to - disable. Consult the curl manpage for --proxy usage. + completely disable or ‘none’ to allow override from + environment. Consult the curl manpage for --proxy usage. """, 'notes': """ @@ -184,6 +184,9 @@ To add a portfolio, edit the file """ }, 'code': { + 'options': lambda s: s.format( + dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), + ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), @@ -196,16 +199,17 @@ To add a portfolio, edit the file } } +import os + from mmgen.cfg import Config -gcfg = Config( opts_data=opts_data, do_post_init=True ) +import mmgen_node_tools.Ticker as tck -import mmgen_node_tools.Ticker as Ticker -Ticker.gcfg = gcfg +tck.gcfg = Config( opts_data=opts_data, do_post_init=True ) -cfg_in = get_cfg_in() +tck.make_cfg() -cfg = make_cfg(gcfg._args,cfg_in) +from .Ticker import cfg_in -gcfg._post_init() +tck.gcfg._post_init() -main(cfg,cfg_in) +tck.main() diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index c4536d6..c1e925d 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -198,7 +198,7 @@ class TestSuiteScripts(TestSuiteBase): [ 'USD BTC CHG_7d CHG_24h UPDATED', r'BITCOIN 23,250.7741 1.0000000000 \+11.15 \+0.89 10 minutes ago', - r'FAKECOIN 81.3008 0.0034966927 -- -- just now', + r'FAKECOIN 81.3008 0.0034966927 -- -- --', r'\(no data for noc-nocoin\)', ]) os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) @@ -247,7 +247,7 @@ class TestSuiteScripts(TestSuiteBase): 'Offer: 200,000 INR', 'Offered price differs from spot by -7.58%', 'SPOT PRICE OFFERED PRICE UPDATED', - 'INDIAN RUPEE 200,000.00000000 184,843.65372424 10 minutes ago ' + + 'INDIAN RUPEE 200,000.00000000 184,843.65372424 -- ' + 'BITCOIN 0.10819955 0.10000000 10 minutes ago' ]) @@ -272,7 +272,7 @@ class TestSuiteScripts(TestSuiteBase): 'Offered price differs from spot by -3.72%', 'SPOT PRICE OFFERED PRICE UPDATED', 'BITCOIN 2.00000000 1.92563954 1 day 9 hours 2 minutes ago ' + - 'US DOLLAR 46,737.71911598 45,000.00000000 1 day 9 hours 2 minutes ago', + 'US DOLLAR 46,737.71911598 45,000.00000000 --', ]) def ticker16(self): @@ -283,7 +283,7 @@ class TestSuiteScripts(TestSuiteBase): r'OMR \(OMANI RIAL\) = 2.5900 USD', 'USD EUR OMR BTC CHG_7d CHG_24h UPDATED', r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', - 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- just now' + 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- --' ]) def ticker17(self): From d29e34c221b8f95099c6f56edb62bf7fc2b0ce95 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 25 Sep 2023 15:53:02 +0000 Subject: [PATCH 029/108] mmnode-ticker: support multiple data sources --- mmgen_node_tools/Ticker.py | 317 ++++++++++++++++++++------------ mmgen_node_tools/main_ticker.py | 29 ++- 2 files changed, 210 insertions(+), 136 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index a370b56..cafd407 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -12,11 +12,6 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets """ -api_host = 'api.coinpaprika.com' -api_url = f'https://{api_host}/v1/ticker' -ratelimit = 240 -btc_ratelimit = 10 - # We use deprecated coinpaprika ‘ticker’ API for now because it returns ~45% less data. # Old ‘ticker’ API (/v1/ticker): data['BTC']['price_usd'] # New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price'] @@ -24,25 +19,193 @@ btc_ratelimit = 10 # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,time,json,yaml +import sys,os,re,time,json,yaml,random from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple from mmgen.color import * -from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list +from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen homedir = os.getenv('HOME') dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' +asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) + +def fetch_delay(fetched_data=[]): + if not gcfg.testing: + if fetched_data: + delay = 1 + random.randrange(1,5000) / 1000 + msg(f'Waiting {delay:.3f} seconds...') + time.sleep(delay) + else: + fetched_data.append(None) + +class DataSource: + + sources = { + 'cc': 'coinpaprika', + } + + class base: + + def get_data_from_network(self): + + curl_cmd = list_gen( + ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], + ['--compressed'], # adds 'Accept-Encoding: gzip' + ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], + ['--silent', not gcfg.verbose], + [self.api_url] + ) + + if gcfg.testing: + Msg(fmt_list(curl_cmd,fmt='bare')) + return + + try: + return run(curl_cmd,check=True,stdout=PIPE).stdout.decode() + except CalledProcessError as e: + msg('') + from .Misc import curl_exit_codes + msg(red(curl_exit_codes[e.returncode])) + msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) + from mmgen.exception import MMGenCalledProcessError + raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') + + def get_data(self): + + if not os.path.exists(cfg.cachedir): + os.makedirs(cfg.cachedir) + + if not os.path.exists(self.json_fn): + open(self.json_fn,'w').write('{}') + + if gcfg.cached_data: + data_type = 'json' + data_in = open(self.json_fn).read() + else: + data_type = self.net_data_type + elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) + if elapsed >= self.timeout: + if gcfg.testing: + msg('') + fetch_delay() + msg_r(f'Fetching data from {self.api_host}...') + if self.has_verbose: + gcfg._util.vmsg('') + data_in = self.get_data_from_network() + msg('done') + if gcfg.testing: + return {} + else: + die(1,self.rate_limit_errmsg(elapsed)) + + if data_type == 'json': + try: + data = json.loads(data_in) + except: + self.json_data_error_msg(data_in) + die(2,'Retrieved data is not valid JSON, exiting') + json_text = data_in + elif data_type == 'python': + data = data_in + json_text = json.dumps(data_in) + + if not data: + if gcfg.cached_data: + die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') + else: + die(2,'Remote host returned no data!') + elif 'error' in data: + die(1,data['error']) + + if gcfg.cached_data: + msg(f'Using cached data from ~/{self.json_fn_rel}') + else: + open(self.json_fn,'w').write(json_text) + msg(f'JSON data cached to ~/{self.json_fn_rel}') + if gcfg.download: + sys.exit(0) + + return self.postprocess_data(data) + + def json_data_error_msg(self,json_text): + pass + + def postprocess_data(self,data): + return data + + @property + def json_fn_rel(self): + return os.path.relpath(self.json_fn,start=homedir) + + class coinpaprika(base): + desc = 'CoinPaprika' + api_host = 'api.coinpaprika.com' + ratelimit = 240 + btc_ratelimit = 10 + net_data_type = 'json' + has_verbose = True + + def rate_limit_errmsg(self,elapsed): + return ( + f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + + ('' if cfg.btc_only else ', or use --cached-data or --btc') + ) + + @property + def api_url(self): + return f'https://{self.api_host}/v1/ticker' + ('/btc-bitcoin' if cfg.btc_only else '') + + @property + def json_fn(self): + return os.path.join( + cfg.cachedir, + 'ticker-btc.json' if cfg.btc_only else 'ticker.json' ) + + @property + def timeout(self): + return 5 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit + + def json_data_error_msg(self,json_text): + tor_captcha_msg = f""" + If you’re using Tor, the API request may have failed due to Captcha protection. + A workaround for this issue is to retrieve the JSON data with a browser from + the following URL: + + {self.api_url} + + and save it to: + + ‘{cfg.cachedir}/ticker.json’ + + Then invoke the program with --cached-data and without --btc + """ + msg(json_text[:1024] + '...') + msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) + + def postprocess_data(self,data): + return [data] if cfg.btc_only else data + + @staticmethod + def parse_asset_id(s,require_label): + sym,label = (*s.split('-',1),None)[:2] + if require_label and not label: + die(1,f'{s!r}: asset label is missing') + return asset_tuple( + symbol = sym.upper(), + id = (s.lower() if label else None), + source = 'cc' ) def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield('') yield(k.upper()) for e in v: - yield(' {:4s} {}'.format(*e.split('-',1))) + out = e.split('-',1) + yield(' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '')) def gen_data(data): """ @@ -59,7 +222,7 @@ def gen_data(data): def dup_sym_errmsg(dup_sym): return ( f'The symbol {dup_sym!r} is shared by the following assets:\n' + - '\n ' + '\n '.join(d['id'] for d in data if d['symbol'] == dup_sym) + + '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + f'instead of {dup_sym!r}' ) @@ -103,13 +266,13 @@ def gen_data(data): wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} - for d in data: + for d in data['cc']: if d['id'] == 'btc-bitcoin': btcusd = Decimal(d['price_usd']) break for k in ('id','symbol'): - for d in data: + for d in data['cc']: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: @@ -149,87 +312,6 @@ def gen_data(data): 'last_updated': None, }) -def get_src_data(curl_cmd): - - tor_captcha_msg = f""" - If you’re using Tor, the API request may have failed due to Captcha protection. - A workaround for this issue is to retrieve the JSON data with a browser from - the following URL: - - {api_url} - - and save it to: - - ‘{cfg.cachedir}/ticker.json’ - - Then invoke the program with --cached-data and without --btc - """ - - def rate_limit_errmsg(timeout,elapsed): - return ( - f'Rate limit exceeded! Retry in {timeout-elapsed} seconds' + - ('' if cfg.btc_only else ', or use --cached-data or --btc') - ) - - if not os.path.exists(cfg.cachedir): - os.makedirs(cfg.cachedir) - - if cfg.btc_only: - fn = os.path.join(cfg.cachedir,'ticker-btc.json') - timeout = 5 if gcfg.test_suite else btc_ratelimit - else: - fn = os.path.join(cfg.cachedir,'ticker.json') - timeout = 5 if gcfg.test_suite else ratelimit - - fn_rel = os.path.relpath(fn,start=homedir) - - if not os.path.exists(fn): - open(fn,'w').write('{}') - - if gcfg.cached_data: - json_text = open(fn).read() - else: - elapsed = int(time.time() - os.stat(fn).st_mtime) - if elapsed >= timeout: - msg_r(f'Fetching data from {api_host}...') - gcfg._util.vmsg('') - try: - cp = run(curl_cmd,check=True,stdout=PIPE) - except CalledProcessError as e: - msg('') - from .Misc import curl_exit_codes - msg(red(curl_exit_codes[e.returncode])) - msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) - from mmgen.exception import MMGenCalledProcessError - raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') - json_text = cp.stdout.decode() - msg('done') - else: - die(1,rate_limit_errmsg(timeout,elapsed)) - - try: - data = json.loads(json_text) - except: - msg(json_text[:1024] + '...') - msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) - die(2,'Retrieved data is not valid JSON, exiting') - - if not data: - if gcfg.cached_data: - die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') - else: - die(2,'Remote host returned no data!') - elif 'error' in data: - die(1,data['error']) - - if gcfg.cached_data: - msg(f'Using cached data from ~/{fn_rel}') - else: - open(fn,'w').write(json_text) - msg(f'JSON data cached to ~/{fn_rel}') - - return data - def main(): def update_sample_file(usr_cfg_file): @@ -243,20 +325,6 @@ def main(): sample_file )) open(sample_file,'w').write(usr_data) - def get_curl_cmd(): - return ([ - 'curl', - '--tr-encoding', - '--compressed', # adds 'Accept-Encoding: gzip' - '--header', 'Accept: application/json', - ] + - (['--proxy', cfg.proxy] if cfg.proxy else []) + - (['--silent'] if not gcfg.verbose else []) + - [api_url + ('/btc-bitcoin' if cfg.btc_only else '')] - ) - - global cfg,cfg_in - try: from importlib.resources import files # Python 3.9 except ImportError: @@ -269,17 +337,24 @@ def main(): die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( os.path.relpath(cfg_in.portfolio_file,start=homedir))) - curl_cmd = get_curl_cmd() + if gcfg.list_ids: + src_ids = ['cc'] + elif gcfg.download: + assert gcfg.download in DataSource.sources, f'{gcfg.download!r}: invalid data source' + src_ids = [gcfg.download] + else: + src_ids = DataSource.sources - if gcfg.print_curl: - Msg(curl_cmd + '\n' + ' '.join(curl_cmd)) + ids = random.sample( list(src_ids), k=len(src_ids) ) # shuffle the ids + + src_data = { k: src_cls[k]().get_data() for k in ids } + + if gcfg.testing: return - src_data = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) - if gcfg.list_ids: from mmgen.ui import do_pager - do_pager('\n'.join(e['id'] for e in src_data)) + do_pager('\n'.join(e['id'] for e in src_data['cc'])) return global now @@ -304,10 +379,7 @@ def make_cfg(): return tuple(gen()) def parse_asset_id(s,require_label=False): - sym,label = (*s.split('-',1),None)[:2] - if require_label and not label: - die(1,f'{s!r}: asset label is missing') - return asset_tuple( sym.upper(), (s.lower() if label else None) ) + return src_cls['cc'].parse_asset_id(s,require_label) def parse_usr_asset_arg(key,use_cf_file=False): """ @@ -327,7 +399,8 @@ def make_cfg(): None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else Decimal(rate) ), - rate_asset = parse_asset_id(rate_asset) if rate_asset else None ) + rate_asset = parse_asset_id(rate_asset) if rate_asset else None, + source = parsed_id.source ) cl_opt = getattr(gcfg,key) cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] @@ -344,7 +417,8 @@ def make_cfg(): id = parsed_id.id, amount = None if amount is None else Decimal(amount), rate = None, - rate_asset = None ) + rate_asset = None, + source = parsed_id.source ) ss = s.split(':') assert len(ss) in (2,3,4), f'{s}: malformed argument' @@ -417,14 +491,15 @@ def make_cfg(): 'proxy', 'portfolio' ]) - global cfg_in,cfg + global cfg_in,src_cls,cfg + + src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } cmd_args = gcfg._args cfg_in = get_cfg_in() query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset']) - asset_tuple = namedtuple('asset_tuple',['symbol','id']) + asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index b8df683..6553d2a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -12,8 +12,6 @@ mmnode-ticker: Display price information for cryptocurrency and other assets """ -from .Ticker import * - opts_data = { 'sets': [ ('wide', True, 'percent_change', True), @@ -24,7 +22,7 @@ opts_data = { 'text': { 'desc': 'Display prices for cryptocurrency and other assets', 'usage': '[opts] [TRADE_SPECIFIER]', - 'options': f""" + 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -A, --adjust=P Adjust prices by percentage ‘P’. In ‘trading’ mode, @@ -37,6 +35,8 @@ opts_data = { live data from server -D, --cachedir=D Read and write cached JSON data to directory ‘D’ instead of ‘~/{dfl_cachedir}’ +-d, --download=D Retrieve data ‘D’ from source, save to file and exit + (valid options: {ds}) -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -112,7 +112,7 @@ A TRADE_SPECIFIER is a single argument in the format: PROXY NOTE -The remote server used to obtain the price data, {api_host!r}, blocks +The remote server used to obtain the price data, {cc.api_host!r}, blocks Tor behind a Captcha wall, so a Tor proxy cannot be used directly. If you’re concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’ @@ -121,7 +121,7 @@ the script directly on the VPN’ed host with ’proxy’ or --proxy set to the null string. Alternatively, you may download the JSON source data in a Tor-proxied browser -from ‘{api_url}’, save it as ‘ticker.json’ in your +from ‘{cc.api_url}’, save it as ‘ticker.json’ in your configured cache directory, and run the script with the --cached-data option. @@ -130,9 +130,9 @@ configured cache directory, and run the script with the --cached-data option. To protect user privacy, all filtering and processing of data is performed client side so that the remote server does not know which assets are being examined. This means that data for ALL available assets (currently over 4000) -is fetched with each invocation of the script. A rate limit of {L} seconds +is fetched with each invocation of the script. A rate limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the remote server. When the ---btc option is in effect, this limit is reduced to {B} seconds. To bypass the +--btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data. @@ -186,21 +186,20 @@ To add a portfolio, edit the file 'code': { 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), + ds = fmt_dict(DataSource.sources,fmt='equal'), ), 'notes': lambda s: s.format( - assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), - cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), - pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), - api_host = api_host, - api_url = api_url, - L = ratelimit, - B = btc_ratelimit, + assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), + cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), + pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + cc = src_cls['cc'](), ) } } import os +from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config import mmgen_node_tools.Ticker as tck @@ -208,7 +207,7 @@ tck.gcfg = Config( opts_data=opts_data, do_post_init=True ) tck.make_cfg() -from .Ticker import cfg_in +from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls tck.gcfg._post_init() From fd318909c21d602701c2a219bedb1aafa8ffb0d8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 25 Sep 2023 15:53:02 +0000 Subject: [PATCH 030/108] mmnode-ticker: retrieve financial data from Yahoo Finance --- mmgen_node_tools/Ticker.py | 96 +++++++++++++++++++++++++-- mmgen_node_tools/data/ticker-cfg.yaml | 23 ++++--- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 65 +++++++++++------- setup.cfg | 3 +- test/ref/ticker/ticker-cfg.yaml | 16 ++--- test/ref/ticker/ticker-finance.json | 1 + test/ref/ticker/ticker.json | 2 +- test/test_py_d/ts_misc.py | 48 ++++++-------- 9 files changed, 179 insertions(+), 77 deletions(-) create mode 100644 test/ref/ticker/ticker-finance.json diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index cafd407..e6f0650 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -46,6 +46,7 @@ class DataSource: sources = { 'cc': 'coinpaprika', + 'fi': 'yahooquery' } class base: @@ -199,6 +200,69 @@ class DataSource: id = (s.lower() if label else None), source = 'cc' ) + class yahooquery(base): + + desc = 'Yahoo Finance' + api_host = 'finance.yahoo.com' + ratelimit = 30 + net_data_type = 'python' + has_verbose = False + asset_id_pat = r'^\^.*|.*=[xf]$' + + @staticmethod + def get_id(sym,data): + return sym.lower() + + @staticmethod + def conv_data(sym,data,btcusd): + price_usd = Decimal( data['regularMarketPrice']['raw'] ) + return { + 'id': sym, + 'name': data['shortName'], + 'symbol': sym.upper(), + 'price_usd': str(price_usd), + 'price_btc': str(price_usd / btcusd), + 'percent_change_7d': None, + 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, + 'last_updated': data['regularMarketTime'], + } + + def rate_limit_errmsg(self,elapsed): + return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' + + @property + def json_fn(self): + return os.path.join( cfg.cachedir, 'ticker-finance.json' ) + + @property + def timeout(self): + return 5 if gcfg.test_suite else self.ratelimit + + def get_data_from_network(self): + + arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + + kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } } + + if gcfg.test_suite: + kwargs.update({ 'timeout': 1, 'retry': 0 }) + + if gcfg.testing: + Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( + arg, + fmt_dict(kwargs,fmt='kwargs') )) + return + + from yahooquery import Ticker + return Ticker(arg,**kwargs).price + + @staticmethod + def parse_asset_id(s,require_label): + return asset_tuple( + symbol = s.upper(), + id = s.lower(), + source = 'fi' ) + def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield('') @@ -271,6 +335,23 @@ def gen_data(data): btcusd = Decimal(d['price_usd']) break + get_id = src_cls['fi'].get_id + conv_func = src_cls['fi'].conv_data + + for k,v in data['fi'].items(): + id = get_id(k,v) + if wants['id']: + if id in wants['id']: + if id in found['id']: + die(1,dup_sym_errmsg(id)) + yield ( id, conv_func(id,v,btcusd) ) + found['id'].add(id) + wants['id'].remove(id) + if id in usr_rate_assets_want['id']: + rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key + else: + break + for k in ('id','symbol'): for d in data['cc']: if wants[k]: @@ -379,7 +460,7 @@ def make_cfg(): return tuple(gen()) def parse_asset_id(s,require_label=False): - return src_cls['cc'].parse_asset_id(s,require_label) + return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) def parse_usr_asset_arg(key,use_cf_file=False): """ @@ -489,11 +570,13 @@ def make_cfg(): 'add_prec', 'cachedir', 'proxy', + 'proxy2', 'portfolio' ]) global cfg_in,src_cls,cfg src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } + fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args cfg_in = get_cfg_in() @@ -514,6 +597,7 @@ def make_cfg(): proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy + proxy2 = get_proxy('proxy2') cfg = cfg_tuple( rows = create_rows(), @@ -526,6 +610,7 @@ def make_cfg(): add_prec = parse_add_precision(gcfg.add_precision), cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, + proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None ) @@ -541,9 +626,12 @@ def get_cfg_in(): cfg = cfg_data or { 'assets': { 'coin': [ 'btc-bitcoin', 'eth-ethereum', 'xmr-monero' ], - 'commodity': [ 'xau-gold-spot-token', 'xag-silver-spot-token', 'xbr-brent-crude-oil-spot' ], - 'fiat': [ 'gbp-pound-sterling-token', 'eur-euro-token' ], - 'index': [ 'dj30-dow-jones-30-token', 'spx-sp-500', 'ndx-nasdaq-100-token' ], + # gold futures, silver futures, Brent futures + 'commodity': [ 'gc=f', 'si=f', 'bz=f' ], + # Pound Sterling, Euro, Swiss Franc + 'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ], + # Dow Jones Industrials, Nasdaq 100, S&P 500 + 'index': [ '^dji', '^ixic', '^gspc' ], }, 'proxy': 'http://vpn-gw:8118' }, diff --git a/mmgen_node_tools/data/ticker-cfg.yaml b/mmgen_node_tools/data/ticker-cfg.yaml index 6589bfc..6344445 100644 --- a/mmgen_node_tools/data/ticker-cfg.yaml +++ b/mmgen_node_tools/data/ticker-cfg.yaml @@ -3,10 +3,16 @@ ### See the curl manpage for supported --proxy parameters ### For a direct connection, leave the right-hand side blank proxy: http://vpn-gw:8118 +# proxy2: http://gw2:8118 ### Override the default cache directory (~/.cache/mmgen-node-tools): cachedir: +### Additional asset columns: +# add_columns: +# - cnhusd=x # Yuan +# - 6j=f # Yen futures + ### Asset rows ### Asset labels are arbitrary strings. Use as many or few as you wish. ### Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. @@ -19,13 +25,14 @@ assets: - ada-cardano - bnb-binance-coin commodity: - - xau-gold-spot-token - - xag-silver-spot-token - - xbr-brent-crude-oil-spot + - gc=f # gold futures + - si=f # silver futures + - bz=f # Brent futures fiat: - - gbp-pound-sterling-token - - eur-euro-token + - gbpusd=x # Pound Sterling + - eurusd=x # Euro + - chfusd=x # Swiss Franc index: - - dj30-dow-jones-30-token - - spx-sp-500 - - ndx-nasdaq-100-token + - ^dji # Dow Jones Industrials + - ^ixic # Nasdaq 100 + - ^gspc # S&P 500 diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index b473ce1..0c84a84 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev0 +3.2.dev1 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 6553d2a..f329e03 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -47,7 +47,7 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. --t, --testing Print command to be executed to stdout and exit +-t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column -v, --verbose Be more verbose @@ -55,6 +55,8 @@ opts_data = { -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to completely disable or ‘none’ to allow override from environment. Consult the curl manpage for --proxy usage. +-X, --proxy2=P Alternate proxy for non-crypto financial data. Defaults + to value of --proxy """, 'notes': """ @@ -69,10 +71,15 @@ price to the spot price. ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids) consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the -symbol is ambiguous, the full ID must be used. Examples: +symbol is ambiguous, the full ID must be used. For Yahoo Finance assets +the symbol and ID are identical: - chf - specify asset by symbol - chf-swiss-franc-token - same as above, but use full ID instead of symbol +Examples: + + ltc - specify asset by symbol + ltc-litecoin - same as above, but use full ID instead of symbol + ^dji - Dow Jones Industrial Average (Yahoo) + gc=f - gold futures (Yahoo) ASSET SPECIFIERS have the following format: @@ -89,7 +96,8 @@ postfixed with the letter ‘r’, its meaning is reversed, i.e. interpreted as inr:0.01257r - same as above, but use reverse rate (INR/USD) inr-indian-rupee:79.5 - same as first example, but add an arbitrary label omr-omani-rial:2.59r - Omani Rial is pegged to the Dollar at 2.59 USD - bgn-bg-lev:0.5113r:eur - Bulgarian Lev is pegged to the Euro at 0.5113 EUR + bgn-bulgarian-lev:0.5113r:eurusd=x + - Bulgarian Lev is pegged to the Euro at 0.5113 EUR A TRADE_SPECIFIER is a single argument in the format: @@ -99,8 +107,8 @@ A TRADE_SPECIFIER is a single argument in the format: xmr:17.34 - price of 17.34 XMR in all configured assets xmr-monero:17.34 - same as above, but with full ID - xmr:17.34:eur - price of 17.34 XMR in EUR only - xmr:17.34:eur:2800 - commission on an offer of 17.34 XMR for 2800 EUR + xmr:17.34:eurusd=x - price of 17.34 XMR in EUR only + xmr:17.34:eurusd=x:2800 - commission on an offer of 17.34 XMR for 2800 EUR TO_AMOUNT, if included, is used to calculate the percentage difference or commission on an offer compared to the spot price. @@ -112,28 +120,34 @@ A TRADE_SPECIFIER is a single argument in the format: PROXY NOTE -The remote server used to obtain the price data, {cc.api_host!r}, blocks -Tor behind a Captcha wall, so a Tor proxy cannot be used directly. If you’re -concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then -set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’ -option in the config file or --proxy on the command line accordingly. Or run -the script directly on the VPN’ed host with ’proxy’ or --proxy set to the -null string. +The remote server used to obtain the crypto price data, {cc.api_host}, +blocks Tor behind a Captcha wall, so a Tor proxy cannot be used directly. +If you’re concerned about privacy, connect via a VPN, or better yet, VPN over +Tor. Then set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the +‘proxy’ option in the config file or --proxy on the command line accordingly. +Or run the script directly on the VPN’ed host with ’proxy’ or --proxy set to +the null string. Alternatively, you may download the JSON source data in a Tor-proxied browser -from ‘{cc.api_url}’, save it as ‘ticker.json’ in your -configured cache directory, and run the script with the --cached-data option. +from {cc.api_url}, save it as ‘ticker.json’ in your +configured cache directory and run the script with the --cached-data option. + +Financial data is obtained from {fi.desc}, which currently allows Tor. RATE LIMITING NOTE -To protect user privacy, all filtering and processing of data is performed -client side so that the remote server does not know which assets are being -examined. This means that data for ALL available assets (currently over 4000) -is fetched with each invocation of the script. A rate limit of {cc.ratelimit} seconds -between calls is thus imposed to prevent abuse of the remote server. When the ---btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the -rate limit entirely, use --cached-data. +To protect user privacy, all filtering and processing of cryptocurrency data +is performed client side so that the remote server does not know which assets +are being examined. This means that data for ALL available crypto assets +(currently over 8000) is fetched with each invocation of the script. A rate +limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the +remote server. When the --btc option is in effect, this limit is reduced to +{cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data. + +Note that financial data obtained from {fi.api_host} is filtered in the +request, which has privacy implications. The rate limit for financial data +is {fi.ratelimit} seconds. EXAMPLES @@ -146,10 +160,10 @@ $ mmnode-ticker --btc # Wide display, add EUR and OMR columns, OMR/USD rate, extra precision and # proxy: -$ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 +$ mmnode-ticker -w -c eurusd=x,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 # Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate: -$ mmnode-ticker -wE -c eur,bgn-bulgarian-lev:0.5113r:eur +$ mmnode-ticker -wE -c eurusd=x,bgn-bulgarian-lev:0.5113r:eurusd=x # Wide display, use cached data from previous network query, show portfolio # (see above), pipe output to pager, add DOGE row: @@ -193,6 +207,7 @@ To add a portfolio, edit the file cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), cc = src_cls['cc'](), + fi = src_cls['fi'](), ) } } diff --git a/setup.cfg b/setup.cfg index 0cdbed1..9539a29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,8 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev44 + mmgen>=14.0.dev2 + yahooquery packages = mmgen_node_tools diff --git a/test/ref/ticker/ticker-cfg.yaml b/test/ref/ticker/ticker-cfg.yaml index 504a7fb..4b40a43 100644 --- a/test/ref/ticker/ticker-cfg.yaml +++ b/test/ref/ticker/ticker-cfg.yaml @@ -12,13 +12,13 @@ assets: - ada-cardano - algo-algorand commodity: - - xau-gold-spot-token - - xag-silver-spot-token - - xbr-brent-crude-oil-spot + - gc=f # gold futures + - si=f # silver futures + - bz=f # Brent futures fiat: - - chf-swiss-franc-token - - eur-euro-token + - chfusd=x # Swiss Franc + - eurusd=x # Euro index: - - dj30-dow-jones-30-token - - spx-sp-500 - - ndx-nasdaq-100-token + - ^dji # Dow Jones Industrials + - ^ixic # Nasdaq 100 + - ^gspc # S&P 500 diff --git a/test/ref/ticker/ticker-finance.json b/test/ref/ticker/ticker-finance.json new file mode 100644 index 0000000..ac0af29 --- /dev/null +++ b/test/ref/ticker/ticker-finance.json @@ -0,0 +1 @@ +{"GC=F":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0015419408,"fmt":"-0.15%"},"regularMarketChange":{"raw":-3,"fmt":"-3.00"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":1942.6,"fmt":"1,942.60"},"regularMarketDayHigh":{"raw":1946.8,"fmt":"1,946.80"},"regularMarketDayLow":{"raw":1940.1,"fmt":"1,940.10"},"regularMarketVolume":{"raw":39539,"fmt":"39.54k","longFmt":"39,539.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1945.6,"fmt":"1,945.60"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1944.7,"fmt":"1,944.70"},"strikePrice":{},"openInterest":{},"exchange":"CMX","exchangeName":"COMEX","exchangeDataDelayedBy":10,"marketState":"REGULAR","quoteType":"FUTURE","symbol":"GC=F","underlyingSymbol":null,"shortName":"Gold Dec 23","longName":null,"currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"SI=F":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0014259518,"fmt":"-0.14%"},"regularMarketChange":{"raw":-0.034000397,"fmt":"-0.03"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":23.81,"fmt":"23.81"},"regularMarketDayHigh":{"raw":23.925,"fmt":"23.92"},"regularMarketDayLow":{"raw":23.67,"fmt":"23.67"},"regularMarketVolume":{"raw":11819,"fmt":"11.82k","longFmt":"11,819.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":23.844,"fmt":"23.84"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":23.81,"fmt":"23.81"},"strikePrice":{},"openInterest":{},"exchange":"CMX","exchangeName":"COMEX","exchangeDataDelayedBy":10,"marketState":"REGULAR","quoteType":"FUTURE","symbol":"SI=F","underlyingSymbol":null,"shortName":"Silver Dec 23","longName":null,"currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"BZ=F":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":0.004610275,"fmt":"0.46%"},"regularMarketChange":{"raw":0.4300003,"fmt":"0.43"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":93.7,"fmt":"93.70"},"regularMarketDayHigh":{"raw":94.25,"fmt":"94.25"},"regularMarketDayLow":{"raw":93.24,"fmt":"93.24"},"regularMarketVolume":{"raw":2853,"fmt":"2.85k","longFmt":"2,853.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":93.27,"fmt":"93.27"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":93.75,"fmt":"93.75"},"strikePrice":{},"openInterest":{},"exchange":"NYM","exchangeName":"NY Mercantile","exchangeDataDelayedBy":10,"marketState":"REGULAR","quoteType":"FUTURE","symbol":"BZ=F","underlyingSymbol":null,"shortName":"Brent Crude Oil Last Day Financ","longName":null,"currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"GBPUSD=X":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.00045271934,"fmt":"-0.0453%"},"regularMarketChange":{"raw":-0.0005540848,"fmt":"-0.0006"},"regularMarketTime":1659385900,"priceHint":{"raw":4,"fmt":"4","longFmt":"4"},"regularMarketPrice":{"raw":1.223481,"fmt":"1.2235"},"regularMarketDayHigh":{"raw":1.2263919,"fmt":"1.2264"},"regularMarketDayLow":{"raw":1.2213293,"fmt":"1.2213"},"regularMarketVolume":{"raw":0,"fmt":null,"longFmt":"0.0000"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1.2240351,"fmt":"1.2240"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1.2240951,"fmt":"1.2241"},"strikePrice":{},"openInterest":{},"exchange":"CCY","exchangeName":"CCY","exchangeDataDelayedBy":0,"marketState":"REGULAR","quoteType":"CURRENCY","symbol":"GBPUSD=X","underlyingSymbol":null,"shortName":"GBP/USD","longName":"GBP/USD","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"EURUSD=X":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0005321096,"fmt":"-0.0532%"},"regularMarketChange":{"raw":-0.00056660175,"fmt":"-0.0006"},"regularMarketTime":1659385900,"priceHint":{"raw":4,"fmt":"4","longFmt":"4"},"regularMarketPrice":{"raw":1.0641694,"fmt":"1.0642"},"regularMarketDayHigh":{"raw":1.0658708,"fmt":"1.0659"},"regularMarketDayLow":{"raw":1.0626993,"fmt":"1.0627"},"regularMarketVolume":{"raw":0,"fmt":null,"longFmt":"0.0000"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1.064736,"fmt":"1.0647"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1.064736,"fmt":"1.0647"},"strikePrice":{},"openInterest":{},"exchange":"CCY","exchangeName":"CCY","exchangeDataDelayedBy":0,"marketState":"REGULAR","quoteType":"CURRENCY","symbol":"EURUSD=X","underlyingSymbol":null,"shortName":"EUR/USD","longName":"EUR/USD","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"CHFUSD=X":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0028495365,"fmt":"-0.2850%"},"regularMarketChange":{"raw":-0.0031440258,"fmt":"-0.0031"},"regularMarketTime":1659385900,"priceHint":{"raw":4,"fmt":"4","longFmt":"4"},"regularMarketPrice":{"raw":1.1002069,"fmt":"1.1002"},"regularMarketDayHigh":{"raw":1.1046062,"fmt":"1.1046"},"regularMarketDayLow":{"raw":1.0982735,"fmt":"1.0983"},"regularMarketVolume":{"raw":0,"fmt":null,"longFmt":"0.0000"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1.1033509,"fmt":"1.1034"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1.1043622,"fmt":"1.1044"},"strikePrice":{},"openInterest":{},"exchange":"CCY","exchangeName":"CCY","exchangeDataDelayedBy":0,"marketState":"REGULAR","quoteType":"CURRENCY","symbol":"CHFUSD=X","underlyingSymbol":null,"shortName":"CHF/USD","longName":"CHF/USD","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"^DJI":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0031276005,"fmt":"-0.31%"},"regularMarketChange":{"raw":-106.55859,"fmt":"-106.56"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":33963.84,"fmt":"33,963.84"},"regularMarketDayHigh":{"raw":34156.15,"fmt":"34,156.15"},"regularMarketDayLow":{"raw":33947.24,"fmt":"33,947.24"},"regularMarketVolume":{"raw":271273859,"fmt":"271.27M","longFmt":"271,273,859.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":34070.4,"fmt":"34,070.40"},"regularMarketSource":"FREE_REALTIME","regularMarketOpen":{"raw":34077.08,"fmt":"34,077.08"},"strikePrice":{},"openInterest":{},"exchange":"DJI","exchangeName":"DJI","exchangeDataDelayedBy":0,"marketState":"PRE","quoteType":"INDEX","symbol":"^DJI","underlyingSymbol":null,"shortName":"Dow Jones Industrial Average","longName":"Dow Jones Industrial Average","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"^IXIC":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.00092206284,"fmt":"-0.09%"},"regularMarketChange":{"raw":-12.193359,"fmt":"-12.19"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":13211.807,"fmt":"13,211.81"},"regularMarketDayHigh":{"raw":13353.22,"fmt":"13,353.22"},"regularMarketDayLow":{"raw":13200.639,"fmt":"13,200.64"},"regularMarketVolume":{"raw":3887623000,"fmt":"3.89B","longFmt":"3,887,623,000.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":13224,"fmt":"13,224.00"},"regularMarketSource":"FREE_REALTIME","regularMarketOpen":{"raw":13287.171,"fmt":"13,287.17"},"strikePrice":{},"openInterest":{},"exchange":"NIM","exchangeName":"Nasdaq GIDS","exchangeDataDelayedBy":0,"marketState":"PRE","quoteType":"INDEX","symbol":"^IXIC","underlyingSymbol":null,"shortName":"NASDAQ Composite","longName":"NASDAQ Composite","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"^GSPC":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0022955984,"fmt":"-0.23%"},"regularMarketChange":{"raw":-9.939941,"fmt":"-9.94"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":4320.06,"fmt":"4,320.06"},"regularMarketDayHigh":{"raw":4357.4,"fmt":"4,357.40"},"regularMarketDayLow":{"raw":4316.49,"fmt":"4,316.49"},"regularMarketVolume":{"raw":2135953000,"fmt":"2.14B","longFmt":"2,135,953,000.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":4330,"fmt":"4,330.00"},"regularMarketSource":"FREE_REALTIME","regularMarketOpen":{"raw":4341.74,"fmt":"4,341.74"},"strikePrice":{},"openInterest":{},"exchange":"SNP","exchangeName":"SNP","exchangeDataDelayedBy":0,"marketState":"PRE","quoteType":"INDEX","symbol":"^GSPC","underlyingSymbol":null,"shortName":"S&P 500","longName":"S&P 500","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}}} diff --git a/test/ref/ticker/ticker.json b/test/ref/ticker/ticker.json index 2484e6d..880a666 100644 --- a/test/ref/ticker/ticker.json +++ b/test/ref/ticker/ticker.json @@ -1 +1 @@ -[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"},{"id":"xau-gold-spot-token","name":"Gold Spot Token","symbol":"XAU","rank":"2634","price_usd":"1767.0700000000002","price_btc":"0.07608887785567188","volume_24h_usd":"11881071.852000002","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.33","percent_change_24h":"-0.19","percent_change_7d":"2.89","last_updated":"1659464759"},{"id":"xag-silver-spot-token","name":"Silver Spot Token","symbol":"XAG","rank":"4753","price_usd":"20.0265","price_btc":"0.0008623279849562342","volume_24h_usd":"445379.34674999997","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.61","percent_change_24h":"-1.54","percent_change_7d":"7.49","last_updated":"1659464759"},{"id":"xbr-brent-crude-oil-spot","name":"Brent Crude Oil Spot","symbol":"XBR","rank":"2616","price_usd":"100.39000000000001","price_btc":"0.0043227277062770015","volume_24h_usd":"4527788.105237802","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.89","percent_change_24h":"0.87","percent_change_7d":"1.02","last_updated":"1659464759"},{"id":"chf-swiss-franc-token","name":"Swiss Franc Token","symbol":"CHF","rank":"3234","price_usd":"1.0453383927173951","price_btc":"0.00004501158713651311","volume_24h_usd":"360978.80080814153","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.08","percent_change_24h":"-0.66","percent_change_7d":"0.65","last_updated":"1659464759"},{"id":"eur-euro-token","name":"Euro Token","symbol":"EUR","rank":"5116","price_usd":"1.0185784743417672","price_btc":"0.00004385932256255119","volume_24h_usd":"40587878.087926075","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"0.05","percent_change_24h":"-0.82","percent_change_7d":"0.6","last_updated":"1659464759"},{"id":"gbp-pound-sterling-token","name":"Pound Sterling Token","symbol":"GBP","rank":"3424","price_usd":"1.2181792934946465","price_btc":"0.00005245400321946659","volume_24h_usd":"5428622.322314565","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.61","percent_change_7d":"1.29","last_updated":"1659464759"},{"id":"dj30-dow-jones-30-token","name":"Dow Jones 30 Token","symbol":"DJ30","rank":"3192","price_usd":"32546","price_btc":"1.401409462381624","volume_24h_usd":"293564.92","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.1","percent_change_24h":"-0.91","percent_change_7d":"2.5","last_updated":"1659464759"},{"id":"spx-sp-500","name":"S&P 500 Token","symbol":"SPX","rank":"3342","price_usd":"4110.6","price_btc":"0.17699974608449287","volume_24h_usd":"780397.41","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"-0.27","percent_change_7d":"4.87","last_updated":"1659464759"},{"id":"ndx-nasdaq-100-token","name":"NASDAQ 100 Token","symbol":"NDX","rank":"5097","price_usd":"12948.199999999999","price_btc":"0.5575410188904857","volume_24h_usd":"11221816.494","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"0.08","percent_change_24h":"0.04","percent_change_7d":"7.21","last_updated":"1659464759"}] +[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"}] diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index c1e925d..28a9e8d 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -87,7 +87,6 @@ class TestSuiteScripts(TestSuiteBase): ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), ('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'), - ('ticker18', 'ticker [--cached-data --wide --elapsed -c eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'), ) } @@ -102,6 +101,7 @@ class TestSuiteScripts(TestSuiteBase): def ticker_setup(self): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) + shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' @@ -123,8 +123,8 @@ class TestSuiteScripts(TestSuiteBase): if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - t.expect('proxy host could not be resolved') - t.req_exit_val = 3 + ret = t.expect(['proxy host could not be resolved','ProxyError']) + t.req_exit_val = 3 if ret == 0 else 1 return t def ticker3(self): @@ -138,15 +138,15 @@ class TestSuiteScripts(TestSuiteBase): def ticker4(self): return self.ticker( - ['--wide','--add-columns=eur,inr-indian-rupee:79.5'], + ['--wide','--add-columns=eurusd=x,inr-indian-rupee:79.5'], [ - r'EUR \(EURO TOKEN\) = 1.0186 USD ' + + r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' + r'INR \(INDIAN RUPEE\) = 0.012579 USD', - 'USD EUR INR BTC CHG_7d CHG_24h UPDATED', + 'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED', 'BITCOIN', - r'ETHEREUM 1,659.66 1,629.3906 131,943.14 0.07146397 \+21.42 \+1.82', - r'MONERO 158.97 156.0734 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', - r'INDIAN RUPEE 0.01 0.0123 1.00 0.00000054 -- --', + r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07146397 \+21.42 \+1.82', + r'MONERO 158.97 149.3870 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', + r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --', ]) def ticker5(self): @@ -213,7 +213,7 @@ class TestSuiteScripts(TestSuiteBase): 'SPOT PRICE', 'BTC 0.11783441', 'XMR 17.23400000', - 'XAU','NDX', + 'GC=F',r'\^IXIC', ]) def ticker11(self): @@ -277,32 +277,22 @@ class TestSuiteScripts(TestSuiteBase): def ticker16(self): return self.ticker( - ['--wide','--elapsed','-c','eur,omr-omani-rial:2.59r'], + ['--wide','--elapsed','-c','eurusd=x,omr-omani-rial:2.59r'], [ - r'EUR \(EURO TOKEN\) = 1.0186 USD ' + + r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' + r'OMR \(OMANI RIAL\) = 2.5900 USD', - 'USD EUR OMR BTC CHG_7d CHG_24h UPDATED', - r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', - 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- --' + 'USD EURUSD=X OMR BTC CHG_7d CHG_24h UPDATED', + r'BITCOIN 23,250.77 21,848.7527 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', + 'OMANI RIAL 2.59 2.4338 1.0000 0.00011139 -- -- --' ]) def ticker17(self): # BGN pegged at 0.5113 EUR return self.ticker( - ['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eur'], + ['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eurusd=x'], [ - r'BGN \(BULGARIAN LEV\) = 0.52080 USD', + r'BGN \(BULGARIAN LEV\) = 0.54411 USD', 'USD BGN BTC CHG_7d CHG_24h UPDATED', - 'BITCOIN 23,250.77 44,644.414 1.00000000', - 'BULGARIAN LEV 0.52 1.000 0.00002240', - ]) - - def ticker18(self): - return self.ticker( - ['--wide','--elapsed','-c','eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'], - [ - r'BGN \(BULGARIAN LEV\) = 0.52080 USD', - 'USD EUR BGN BTC CHG_7d CHG_24h UPDATED', - 'BITCOIN 23,250.77 22,826.6890 44,644.414 1.00000000', - 'BULGARIAN LEV 0.52 0.5113 1.000 0.00002240', + 'BITCOIN 23,250.77 42,731.767 1.00000000', + 'BULGARIAN LEV 0.54 1.000 0.00002340', ]) From df620f923f472ef9d03fd3673395511e8b473c8b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 3 Oct 2023 14:35:57 +0000 Subject: [PATCH 031/108] minor fixes and cleanups --- mmgen_node_tools/Sound.py | 5 ++++- mmgen_node_tools/Ticker.py | 28 +++++++++++++++------------- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 24 ++++++++++++------------ mmgen_node_tools/main_txfind.py | 2 +- setup.cfg | 2 +- test/test_py_d/ts_main.py | 7 +++---- test/test_py_d/ts_misc.py | 8 ++++---- test/test_py_d/ts_regtest.py | 8 ++++---- test/unit_tests_d/ut_BlocksInfo.py | 4 +--- 10 files changed, 46 insertions(+), 44 deletions(-) diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index 271253c..88d3b02 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -20,7 +20,10 @@ mmgen_node_tools.Sound: audio-related functions for MMGen node tools """ import sys,os,time -from mmgen_node_tools.Util import * + +from mmgen.util import die + +from mmgen_node_tools.Util import do_system _alsa_config_file = '/tmp/alsa-config-' + os.path.basename(sys.argv[0]) _dvols = { 'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190 } diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index e6f0650..97b5dc5 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import * +from mmgen.color import red,yellow,green,blue,orange,gray from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen homedir = os.getenv('HOME') @@ -37,8 +37,9 @@ def fetch_delay(fetched_data=[]): if not gcfg.testing: if fetched_data: delay = 1 + random.randrange(1,5000) / 1000 - msg(f'Waiting {delay:.3f} seconds...') + msg_r(f'Waiting {delay:.3f} seconds...') time.sleep(delay) + msg('') else: fetched_data.append(None) @@ -265,11 +266,11 @@ class DataSource: def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): - yield('') - yield(k.upper()) + yield '' + yield k.upper() for e in v: out = e.split('-',1) - yield(' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '')) + yield ' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '') def gen_data(data): """ @@ -421,7 +422,8 @@ def main(): if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: - assert gcfg.download in DataSource.sources, f'{gcfg.download!r}: invalid data source' + if not gcfg.download in DataSource.sources: + die(1,f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: src_ids = DataSource.sources @@ -449,19 +451,22 @@ def main(): def make_cfg(): + query_tuple = namedtuple('query',['asset','to_asset']) + asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) + + def parse_asset_id(s,require_label=False): + return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) + def get_rows_from_cfg(add_data=None): def gen(): for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): - yield(k) + yield k if add_data and k in add_data: v += tuple(add_data[k]) for e in v: yield parse_asset_id(e,require_label=True) return tuple(gen()) - def parse_asset_id(s,require_label=False): - return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) - def parse_usr_asset_arg(key,use_cf_file=False): """ asset_id[:rate[:rate_asset]] @@ -581,9 +586,6 @@ def make_cfg(): cmd_args = gcfg._args cfg_in = get_cfg_in() - query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) - usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0c84a84..4b653e3 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev1 +3.2.dev2 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 72692fc..7bee773 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -145,25 +145,25 @@ def create_data(coin_amt,mempool): def gen_header(host,mempool,blockcount): - yield(fmt(f""" + yield fmt(f""" Mempool Fee Structure Date: {make_timestr()} UTC Host: {host} Network: {proto.coin.upper()} {proto.network.upper()} Block: {blockcount} TX count: {len(mempool)} - """)).strip() + """).strip() if cfg.show_empty: - yield('Displaying all fee brackets') + yield 'Displaying all fee brackets' elif cfg.ignore_below: - yield('Ignoring fee brackets with less than {:,} bytes ({})'.format( - ignore_below, - int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), - )) + yield 'Ignoring fee brackets with less than {:,} bytes ({})'.format( + ignore_below, + int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), + ) if cfg.include_current: - yield('Including transactions in current fee bracket in Total MB amounts') + yield 'Including transactions in current fee bracket in Total MB amounts' def fmt_mb(n): return int2bytespec(n,'MB',f'0.{precision}',print_sym=False) @@ -187,17 +187,17 @@ def gen_body(data): for i in data: if not i.skip: cum_bytes = i.tx_bytes_cum + i.tx_bytes if cfg.include_current else i.tx_bytes_cum - yield(fs.format( + yield fs.format( a = '{}-{}'.format(i.bottom,i.top) if cfg.ranges else i.top, b = fmt_mb(i.tx_bytes), c = fmt_mb(cum_bytes), - d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) )) + d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) ) - yield(fs.format( + yield fs.format( a = 'TOTAL', b = '', c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes if data else 0), - d = '' )) + d = '' ) async def main(): diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index cc8df14..57b691a 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -21,7 +21,7 @@ mmnode-txfind: Find a transaction in the blockchain or mempool """ from mmgen.cfg import Config -from mmgen.util import msg,Msg,die +from mmgen.util import msg,Msg,die,is_hex_str opts_data = { 'text': { diff --git a/setup.cfg b/setup.cfg index 9539a29..aeebff5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=14.0.dev2 + mmgen>=14.0.dev6 yahooquery packages = diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index 162e3fb..903981c 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -12,11 +12,10 @@ test_py_d.ts_main: Basic operations tests for the test.py test suite """ -import time +import sys,time -from ..include.common import * -from .common import * -from .ts_base import * +from ..include.common import cfg +from .ts_base import TestSuiteBase class TestSuiteMain(TestSuiteBase): 'basic operations with fake RPC data' diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 28a9e8d..06674ed 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -12,10 +12,10 @@ test.test_py_d.ts_misc: Miscellaneous test groups for the test.py test suite """ -import shutil -from ..include.common import * -from .common import * -from .ts_base import * +import os,shutil + +from ..include.common import cfg +from .ts_base import TestSuiteBase refdir = os.path.join('test','ref','ticker') diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index f8859b8..2ed45b4 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -13,13 +13,13 @@ test.test_py_d.ts_regtest: Regtest tests for the test.py test suite """ import os -from mmgen.util import die,gmsg + +from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest -from ..include.common import * -from .common import * -from .ts_base import * +from ..include.common import cfg,imsg,stop_test_daemons,joinpath +from .ts_base import TestSuiteBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/unit_tests_d/ut_BlocksInfo.py index e65e314..b301a66 100755 --- a/test/unit_tests_d/ut_BlocksInfo.py +++ b/test/unit_tests_d/ut_BlocksInfo.py @@ -3,11 +3,9 @@ test.unit_tests_d.nt_BlocksInfo: BlocksInfo unit test for the MMGen Node Tools suite """ -from mmgen.common import * -from mmgen.exception import * from mmgen_node_tools.BlocksInfo import BlocksInfo -from ..include.common import cfg,vmsg +from ..include.common import vmsg tip = 50000 range_vecs = ( From 666b27c04200ff4312b04a857af0ab9ea88c2518 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 13 Oct 2023 09:50:15 +0000 Subject: [PATCH 032/108] test rename: test.py -> cmdtest.py --- test/{test_py_d => cmdtest_py_d}/cfg.py | 10 +-- .../ts_main.py => cmdtest_py_d/ct_main.py} | 6 +- .../ts_misc.py => cmdtest_py_d/ct_misc.py} | 8 +-- .../ct_regtest.py} | 8 +-- test/init.sh | 65 ++++++++++++------- test/test-release.d/cfg.sh | 12 ++-- 6 files changed, 64 insertions(+), 45 deletions(-) rename test/{test_py_d => cmdtest_py_d}/cfg.py (67%) rename test/{test_py_d/ts_main.py => cmdtest_py_d/ct_main.py} (93%) rename test/{test_py_d/ts_misc.py => cmdtest_py_d/ct_misc.py} (97%) rename test/{test_py_d/ts_regtest.py => cmdtest_py_d/ct_regtest.py} (98%) diff --git a/test/test_py_d/cfg.py b/test/cmdtest_py_d/cfg.py similarity index 67% rename from test/test_py_d/cfg.py rename to test/cmdtest_py_d/cfg.py index c2c72e3..5aacbf9 100755 --- a/test/test_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -9,16 +9,16 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.test_py_d.cfg: configuration data for test.py +test.cmdtest_py_d.cfg: configuration data for cmdtest.py """ import os cmd_groups_dfl = { - 'main': ('TestSuiteMain',{}), - 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), - 'scripts': ('TestSuiteScripts',{'modname':'misc'}), - 'regtest': ('TestSuiteRegtest',{}), + 'main': ('CmdTestMain',{}), + 'helpscreens': ('CmdTestHelp',{'modname':'misc','full_data':True}), + 'scripts': ('CmdTestScripts',{'modname':'misc'}), + 'regtest': ('CmdTestRegtest',{}), } cmd_groups_extra = {} diff --git a/test/test_py_d/ts_main.py b/test/cmdtest_py_d/ct_main.py similarity index 93% rename from test/test_py_d/ts_main.py rename to test/cmdtest_py_d/ct_main.py index 903981c..ad1ce41 100755 --- a/test/test_py_d/ts_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -9,15 +9,15 @@ # https://gitlab.com/mmgen/mmgen """ -test_py_d.ts_main: Basic operations tests for the test.py test suite +cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite """ import sys,time from ..include.common import cfg -from .ts_base import TestSuiteBase +from .ct_base import CmdTestBase -class TestSuiteMain(TestSuiteBase): +class CmdTestMain(CmdTestBase): 'basic operations with fake RPC data' tmpdir_nums = [3] networks = ('btc',) # fake data, so test peerblocks for BTC mainnet only diff --git a/test/test_py_d/ts_misc.py b/test/cmdtest_py_d/ct_misc.py similarity index 97% rename from test/test_py_d/ts_misc.py rename to test/cmdtest_py_d/ct_misc.py index 06674ed..966fb91 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -9,17 +9,17 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.test_py_d.ts_misc: Miscellaneous test groups for the test.py test suite +test.cmdtest_py_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite """ import os,shutil from ..include.common import cfg -from .ts_base import TestSuiteBase +from .ct_base import CmdTestBase refdir = os.path.join('test','ref','ticker') -class TestSuiteHelp(TestSuiteBase): +class CmdTestHelp(CmdTestBase): 'help, info and usage screens' networks = ('btc','ltc','bch') tmpdir_nums = [] @@ -52,7 +52,7 @@ class TestSuiteHelp(TestSuiteBase): def longhelpscreens(self): return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') -class TestSuiteScripts(TestSuiteBase): +class CmdTestScripts(CmdTestBase): 'scripts not requiring a coin daemon' networks = ('btc',) tmpdir_nums = [2] diff --git a/test/test_py_d/ts_regtest.py b/test/cmdtest_py_d/ct_regtest.py similarity index 98% rename from test/test_py_d/ts_regtest.py rename to test/cmdtest_py_d/ct_regtest.py index 2ed45b4..a03e3ff 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.test_py_d.ts_regtest: Regtest tests for the test.py test suite +test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import os @@ -19,7 +19,7 @@ from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest from ..include.common import cfg,imsg,stop_test_daemons,joinpath -from .ts_base import TestSuiteBase +from .ct_base import CmdTestBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] @@ -31,7 +31,7 @@ def gen_addrs(proto,network,keys): tool.addrtype = proto.mmtypes[-1] return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] -class TestSuiteRegtest(TestSuiteBase): +class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' networks = ('btc','ltc','bch') passthru_opts = ('coin',) @@ -104,7 +104,7 @@ class TestSuiteRegtest(TestSuiteBase): } def __init__(self,trunner,cfgs,spawn): - TestSuiteBase.__init__(self,trunner,cfgs,spawn) + CmdTestBase.__init__(self,trunner,cfgs,spawn) if trunner == None: return if self.proto.testnet: diff --git a/test/init.sh b/test/init.sh index a8d03a9..913ac9c 100755 --- a/test/init.sh +++ b/test/init.sh @@ -45,37 +45,56 @@ build_mmgen_extmod() { } create_dir_links() { - for target in 'mmgen' 'scripts'; do - src="$mm_repo/$target" - if [ -e $target ]; then - [ $(realpath --relative-to=. $target) == $src ] || die "'$target' does not point to '$src'" + for link_name in 'mmgen' 'scripts'; do + target="$mm_repo/$link_name" + if [ -e $link_name ]; then + [ $(realpath --relative-to=. $link_name) == $target ] || die "'$link_name' does not point to '$target'" else - echo "Creating symlink: $target" - ln -s $src + echo "Creating symlink: $link_name" + ln -s $target fi done } create_test_links() { - sources=' - test/include - test/overlay/__init__.py - test/overlay/fakemods/mmgen - test/__init__.py - test/test.py - test/unit_tests.py - test/test-release.sh - test/test_py_d/common.py - test/test_py_d/ts_base.py - cmds/mmgen-regtest + paths=' + test/include symbolic + test/overlay/__init__.py symbolic + test/overlay/fakemods/mmgen symbolic + test/__init__.py symbolic + test/cmdtest.py hard + test/unit_tests.py hard + test/test-release.sh symbolic + test/cmdtest_py_d/common.py symbolic + test/cmdtest_py_d/ct_base.py symbolic + cmds/mmgen-regtest symbolic ' - for src in $sources; do - pfx=$(echo $src | sed -r 's/[^/]//g' | sed 's/\//..\//g') - if [ ! -e $src ]; then - echo "Creating symlink: $src" - ( cd "$(dirname $src)" && ln -s "$pfx$mm_repo/$src" ) + while read path type; do + [ "$path" ] || continue + pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') + symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) + target="$mm_repo/$path" + if [ ! -e "$target" ]; then + echo "Target path $target is missing! Cannot proceed" + exit 1 fi - done + fs="%-8s %-16s %s -> %s\n" + if [ $type == 'hard' ]; then + if [ -L $path ]; then + printf "$fs" "Deleting" "symbolic link:" $path $target + rm -rf $path + elif [ -e $path ]; then + if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then + printf "$fs" "Deleting" "stale hard link:" $path "?" + rm -rf $path + fi + fi + fi + if [ ! -e $path ]; then # link is either absent or a broken symlink + printf "$fs" "Creating" "$type link:" $path $target + ( cd "$(dirname $path)" && ln -f $symlink_arg $pfx$target ) + fi + done <<<$paths } set -e diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index cfe98b7..54d8edd 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -41,20 +41,20 @@ init_tests() { t_unit="- $unit_tests_py" d_misc="miscellaneous features" - t_misc="- $test_py helpscreens" + t_misc="- $cmdtest_py helpscreens" d_scripts="scripts not requiring a coin daemon" - t_scripts="- $test_py scripts" + t_scripts="- $cmdtest_py scripts" d_btc="Bitcoin with emulated RPC data" - t_btc="- $test_py main" + t_btc="- $cmdtest_py main" d_btc_rt="Bitcoin regtest" - t_btc_rt="- $test_py regtest" + t_btc_rt="- $cmdtest_py regtest" d_bch_rt="Bitcoin Cash Node (BCH) regtest" - t_bch_rt="- $test_py --coin=bch regtest" + t_bch_rt="- $cmdtest_py --coin=bch regtest" d_ltc_rt="Litecoin regtest" - t_ltc_rt="- $test_py --coin=ltc regtest" + t_ltc_rt="- $cmdtest_py --coin=ltc regtest" } From 3f76be1ab8e948a341688522573767f17fe5f9aa Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 13 Oct 2023 09:50:15 +0000 Subject: [PATCH 033/108] minor fixes and cleanups --- mmgen_node_tools/BlocksInfo.py | 93 +++++++++++++++++--------------- mmgen_node_tools/PollDisplay.py | 2 +- mmgen_node_tools/Ticker.py | 6 ++- mmgen_node_tools/main_addrbal.py | 2 + mmgen_node_tools/main_netrate.py | 4 +- mmgen_node_tools/main_ticker.py | 10 ++-- mmgen_node_tools/main_txfind.py | 4 +- setup.cfg | 1 + test/cmdtest_py_d/ct_regtest.py | 2 +- 9 files changed, 68 insertions(+), 56 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 25e1caf..40cc6e5 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -27,6 +27,52 @@ from time import strftime,gmtime from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int from mmgen.rpc import json_encoder +class RangeParser: + + debug = False + + def __init__(self,caller,arg): + self.caller = caller + self.arg = self.orig_arg = arg + + def parse(self,target): + ret = getattr(self,'parse_'+target)() + if self.debug: + msg(f'arg after parse({target}): {self.arg}') + return ret + + def finalize(self): + if self.arg: + die(1,f'{self.orig_arg!r}: invalid range specifier') + + def parse_from_tip(self): + m = re.match(r'-([0-9]+)(.*)',self.arg) + if m: + res,self.arg = (m[1],m[2]) + return self.caller.check_nblocks(int(res)) + + def parse_abs_range(self): + m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',self.arg) + if m: + if self.debug: + msg(f'abs_range parse: first={m[1]}, last={m[3]}') + self.arg = m[4] + return ( + self.caller.conv_blkspec(m[1]), + self.caller.conv_blkspec(m[3]) if m[3] else None + ) + return (None,None) + + def parse_add(self): + m = re.match(r'\+([0-9*]+)(.*)',self.arg) + if m: + res,self.arg = (m[1],m[2]) + if res.strip('*') != res: + die(1,f"'+{res}': malformed nBlocks specifier") + if len(res) > 30: + die(1,f"'+{res}': overly long nBlocks specifier") + return self.caller.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe + class BlocksInfo: total_bytes = 0 @@ -298,49 +344,8 @@ class BlocksInfo: def parse_rangespec(self,arg): - class RangeParser: - debug = False + p = RangeParser(self,arg) - def __init__(rp,arg): - rp.arg = rp.orig_arg = arg - - def parse(rp,target): - ret = getattr(rp,'parse_'+target)() - if rp.debug: msg(f'arg after parse({target}): {rp.arg}') - return ret - - def finalize(rp): - if rp.arg: - die(1,f'{rp.orig_arg!r}: invalid range specifier') - - def parse_from_tip(rp): - m = re.match(r'-([0-9]+)(.*)',rp.arg) - if m: - res,rp.arg = (m[1],m[2]) - return self.check_nblocks(int(res)) - - def parse_abs_range(rp): - m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',rp.arg) - if m: - if rp.debug: msg(f'abs_range parse: first={m[1]}, last={m[3]}') - rp.arg = m[4] - return ( - self.conv_blkspec(m[1]), - self.conv_blkspec(m[3]) if m[3] else None - ) - return (None,None) - - def parse_add(rp): - m = re.match(r'\+([0-9*]+)(.*)',rp.arg) - if m: - res,rp.arg = (m[1],m[2]) - if res.strip('*') != res: - die(1,f"'+{res}': malformed nBlocks specifier") - if len(res) > 30: - die(1,f"'+{res}': overly long nBlocks specifier") - return self.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe - - p = RangeParser(arg) from_tip = p.parse('from_tip') first,last = (self.tip-from_tip,None) if from_tip else p.parse('abs_range') add1 = p.parse('add') @@ -708,8 +713,8 @@ class BlocksInfo: class JSONBlocksInfo(BlocksInfo): - def __init__(self,cfg,cmd_args,opt,rpc): - super().__init__(cfg,cmd_args,opt,rpc) + def __init__(self,cfg,cmd_args,rpc): + super().__init__(cfg,cmd_args,rpc) if self.cfg.json_raw: self.output_block = self.output_block_raw self.fmt_stat_item = self.fmt_stat_item_raw diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 84a6023..5bbfecd 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -16,7 +16,7 @@ import sys,threading from mmgen.util import msg from mmgen.term import get_char -class PollDisplay(): +class PollDisplay: info = None input = None diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 97b5dc5..1e21192 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -449,7 +449,7 @@ def main(): '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n' ) -def make_cfg(): +def make_cfg(gcfg_arg): query_tuple = namedtuple('query',['asset','to_asset']) asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) @@ -578,7 +578,9 @@ def make_cfg(): 'proxy2', 'portfolio' ]) - global cfg_in,src_cls,cfg + global gcfg,cfg_in,src_cls,cfg + + gcfg = gcfg_arg src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } fi_pat = src_cls['fi'].asset_id_pat diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index a461e9e..34c6236 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -12,6 +12,8 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain """ +import sys + from mmgen.obj import CoinTxID,Int from mmgen.cfg import Config from mmgen.util import msg,Msg,die,suf,make_timestr,async_run diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index f1661c5..52b509d 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -49,7 +49,7 @@ async def main(): d = await c.call('getnettotals') return [float(e) for e in (d['totalbytesrecv'],d['totalbytessent'],d['timemillis'])] - rs = None + rs,ss,ts = (None,None,None) while True: r,s,t = await get_data() @@ -64,7 +64,7 @@ async def main(): if rs is not None: sys.stderr.write('{}{}{}'.format(ERASE_LINE,CUR_UP,ERASE_LINE)) - rs,ss,ts = r,s,t + rs,ss,ts = (r,s,t) try: async_run(main()) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index f329e03..6a9d3aa 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -216,14 +216,14 @@ import os from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config -import mmgen_node_tools.Ticker as tck +from . import Ticker -tck.gcfg = Config( opts_data=opts_data, do_post_init=True ) +gcfg = Config( opts_data=opts_data, do_post_init=True ) -tck.make_cfg() +Ticker.make_cfg(gcfg) from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls -tck.gcfg._post_init() +gcfg._post_init() -tck.main() +Ticker.main() diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 57b691a..4a54430 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -20,8 +20,10 @@ mmnode-txfind: Find a transaction in the blockchain or mempool """ +import sys + from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,is_hex_str +from mmgen.util import msg,Msg,die,is_hex_str,async_run opts_data = { 'text': { diff --git a/setup.cfg b/setup.cfg index aeebff5..7e91ea5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ include_package_data = True install_requires = mmgen>=14.0.dev6 + pyyaml yahooquery packages = diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index a03e3ff..b17b4c6 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -107,7 +107,7 @@ class CmdTestRegtest(CmdTestBase): CmdTestBase.__init__(self,trunner,cfgs,spawn) if trunner == None: return - if self.proto.testnet: + if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) From ada2cecadae5807678f27d34f8a871a49cfa9c65 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 13 Oct 2023 09:50:16 +0000 Subject: [PATCH 034/108] Check the MMGen Node Tools with the pylint static code analyzer: # Install pylint: $ python3 -m pip install pylint # Perform the check: $ test/test-release.sh lint The linting operation should complete without error if running Python v3.9 or greater. --- mmgen_node_tools/data/version | 2 +- pyproject.toml | 22 ++++++++++++++++++++++ setup.cfg | 2 +- test/test-release.d/cfg.sh | 16 ++++++++++++---- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4b653e3..5223fbd 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev2 +3.2.dev3 diff --git a/pyproject.toml b/pyproject.toml index 374b58c..81515d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,25 @@ requires = [ "wheel" ] build-backend = "setuptools.build_meta" + +[tool.pylint.format] +indent-string = "\t" +indent-after-paren = 2 +max-line-length = 110 + +[tool.pylint.main] +py-version = "3.7" +recursive = true +jobs = 0 + +[tool.pylint."messages control"] +ignored-modules = [ + "mmgen.term", + "mmgen.color", +] +ignored-classes = [ + "mmgen_node_tools.Ticker.Ticker.base", + "mmgen_node_tools.Ticker.DataSource.base", + "mmgen_node_tools.PeerBlocks.Display", + "mmgen_node_tools.PollDisplay.PollDisplay", +] diff --git a/setup.cfg b/setup.cfg index 7e91ea5..55c315a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=14.0.dev6 + mmgen>=14.0.dev9 pyyaml yahooquery diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 54d8edd..b371e72 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -18,7 +18,7 @@ # mmnode-ticker OK # mmnode-txfind - -all_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' +all_tests='unit lint misc scripts btc btc_rt bch_rt ltc_rt' groups_desc=" default - All tests minus the extra tests @@ -29,14 +29,22 @@ groups_desc=" " init_groups() { - dfl_tests=$all_tests - extra_tests='' + dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + extra_tests='lint' noalt_tests='unit misc scripts btc btc_rt' quick_tests='unit misc scripts btc btc_rt' - qskip_tests='bch_rt ltc_rt' + qskip_tests='lint bch_rt ltc_rt' } init_tests() { + + d_lint="code errors with static code analyzer" + t_lint=" + - $pylint --errors-only mmgen_node_tools + - $pylint --errors-only test + - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_py_d + " + d_unit="low-level subsystems" t_unit="- $unit_tests_py" From e94f036c73abe57b41e1ffbf3f715198249fddda Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:22 +0000 Subject: [PATCH 035/108] mmnode-ticker: cleanups --- mmgen_node_tools/Ticker.py | 80 ++++++++++++++++++++------------- mmgen_node_tools/main_ticker.py | 14 +++--- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 1e21192..fc0b8da 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -32,26 +32,34 @@ dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) - -def fetch_delay(fetched_data=[]): - if not gcfg.testing: - if fetched_data: - delay = 1 + random.randrange(1,5000) / 1000 - msg_r(f'Waiting {delay:.3f} seconds...') - time.sleep(delay) - msg('') - else: - fetched_data.append(None) +last_api_host = None class DataSource: - sources = { - 'cc': 'coinpaprika', - 'fi': 'yahooquery' - } + source_groups = [ + { + 'cc': 'coinpaprika' + }, { + 'fi': 'yahoospot', + } + ] + + @classmethod + def get_sources(cls,randomize=False): + g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups + return {k:v for a in g for k,v in a.items()} class base: + def fetch_delay(self): + global last_api_host + if not gcfg.testing and last_api_host and last_api_host != self.api_host: + delay = 1 + random.randrange(1,5000) / 1000 + msg_r(f'Waiting {delay:.3f} seconds...') + time.sleep(delay) + msg('') + last_api_host = self.api_host + def get_data_from_network(self): curl_cmd = list_gen( @@ -90,11 +98,11 @@ class DataSource: else: data_type = self.net_data_type elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) - if elapsed >= self.timeout: + if elapsed >= self.timeout or gcfg.testing: if gcfg.testing: msg('') - fetch_delay() - msg_r(f'Fetching data from {self.api_host}...') + self.fetch_delay() + msg_r(f'Fetching {self.data_desc} from {self.api_host}...') if self.has_verbose: gcfg._util.vmsg('') data_in = self.get_data_from_network() @@ -145,6 +153,7 @@ class DataSource: class coinpaprika(base): desc = 'CoinPaprika' + data_desc = 'cryptocurrency data' api_host = 'api.coinpaprika.com' ratelimit = 240 btc_ratelimit = 10 @@ -201,14 +210,16 @@ class DataSource: id = (s.lower() if label else None), source = 'cc' ) - class yahooquery(base): + class yahoospot(base): desc = 'Yahoo Finance' + data_desc = 'spot financial data' api_host = 'finance.yahoo.com' ratelimit = 30 net_data_type = 'python' has_verbose = False asset_id_pat = r'^\^.*|.*=[xf]$' + json_fn_basename = 'ticker-finance.json' @staticmethod def get_id(sym,data): @@ -233,29 +244,37 @@ class DataSource: @property def json_fn(self): - return os.path.join( cfg.cachedir, 'ticker-finance.json' ) + return os.path.join( cfg.cachedir, self.json_fn_basename ) @property def timeout(self): return 5 if gcfg.test_suite else self.ratelimit + @property + def symbols(self): + return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + def get_data_from_network(self): - arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] - - kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } } + kwargs = { + 'formatted': True, + 'proxies': { 'https': cfg.proxy2 }, + } if gcfg.test_suite: kwargs.update({ 'timeout': 1, 'retry': 0 }) if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( - arg, + self.symbols, fmt_dict(kwargs,fmt='kwargs') )) return from yahooquery import Ticker - return Ticker(arg,**kwargs).price + return self.process_network_data( Ticker(self.symbols,**kwargs) ) + + def process_network_data(self,ticker): + return ticker.price @staticmethod def parse_asset_id(s,require_label): @@ -422,15 +441,13 @@ def main(): if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: - if not gcfg.download in DataSource.sources: + if not gcfg.download in DataSource.get_sources(): die(1,f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: - src_ids = DataSource.sources + src_ids = DataSource.get_sources(randomize=True) - ids = random.sample( list(src_ids), k=len(src_ids) ) # shuffle the ids - - src_data = { k: src_cls[k]().get_data() for k in ids } + src_data = { k: src_cls[k]().get_data() for k in src_ids } if gcfg.testing: return @@ -582,7 +599,7 @@ def make_cfg(gcfg_arg): gcfg = gcfg_arg - src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } + src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() } fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args @@ -673,8 +690,7 @@ class Ticker: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) # ticker API - # t.replace('T',' ').replace('Z','') # tickers API + fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) d = self.data max_w = 0 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 6a9d3aa..13175b8 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -14,10 +14,10 @@ mmnode-ticker: Display price information for cryptocurrency and other assets opts_data = { 'sets': [ - ('wide', True, 'percent_change', True), - ('wide', True, 'name_labels', True), - ('wide', True, 'thousands_comma', True), - ('wide', True, 'update_time', True), + ('wide', True, 'percent_change', True), + ('wide', True, 'name_labels', True), + ('wide', True, 'thousands_comma', True), + ('wide', True, 'update_time', True), ], 'text': { 'desc': 'Display prices for cryptocurrency and other assets', @@ -35,8 +35,8 @@ opts_data = { live data from server -D, --cachedir=D Read and write cached JSON data to directory ‘D’ instead of ‘~/{dfl_cachedir}’ --d, --download=D Retrieve data ‘D’ from source, save to file and exit - (valid options: {ds}) +-d, --download=D Retrieve and cache asset data ‘D’ from network (valid + options: {ds}) -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -200,7 +200,7 @@ To add a portfolio, edit the file 'code': { 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), - ds = fmt_dict(DataSource.sources,fmt='equal'), + ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), From 94c61d680702cc1139d75c58f3105cad1b05446b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 036/108] mmnode-ticker: convert amounts to type Decimal on input --- mmgen_node_tools/Ticker.py | 48 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index fc0b8da..b2fc86a 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -232,8 +232,8 @@ class DataSource: 'id': sym, 'name': data['shortName'], 'symbol': sym.upper(), - 'price_usd': str(price_usd), - 'price_btc': str(price_usd / btcusd), + 'price_usd': price_usd, + 'price_btc': price_usd / btcusd, 'percent_change_7d': None, 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, 'last_updated': data['regularMarketTime'], @@ -378,6 +378,13 @@ def gen_data(data): if d[k] in wants[k]: if d[k] in found[k]: die(1,dup_sym_errmsg(d[k])) + if not d.get('_converted'): + d['price_usd'] = Decimal(d['price_usd']) + d['price_btc'] = Decimal(d['price_btc']) + d['percent_change_7d'] = Decimal(d['percent_change_7d']) + d['percent_change_24h'] = Decimal(d['percent_change_24h']) + d['last_updated'] = int(d['last_updated']) + d['_converted'] = True yield (d['id'],d) found[k].add(d[k]) wants[k].remove(d[k]) @@ -394,13 +401,13 @@ def gen_data(data): User-supplied rate overrides rate from source data. """ _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() - ra_rate = Decimal(rate_assets[asset.rate_asset.symbol]['price_usd']) if asset.rate_asset else 1 + ra_rate = rate_assets[asset.rate_asset.symbol]['price_usd'] if asset.rate_asset else 1 yield ( _id, { 'symbol': asset.symbol, 'id': _id, 'name': ' '.join(_id.split('-')[1:]), - 'price_usd': str(Decimal(ra_rate/asset.rate)), - 'price_btc': str(Decimal(ra_rate/asset.rate/btcusd)), + 'price_usd': ra_rate / asset.rate, + 'price_btc': ra_rate / asset.rate / btcusd, 'last_updated': None, }) @@ -408,8 +415,8 @@ def gen_data(data): 'symbol': 'USD', 'id': 'usd-us-dollar', 'name': 'US Dollar', - 'price_usd': '1.0', - 'price_btc': str(Decimal(1/btcusd)), + 'price_usd': Decimal(1), + 'price_btc': Decimal(1) / btcusd, 'last_updated': None, }) @@ -678,7 +685,7 @@ class Ticker: )) + 1 self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows] - self.col_usd_prices = {k:Decimal(self.data[k]['price_usd']) for k in self.col_ids} + self.col_usd_prices = {k:self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id:self.get_row_prices(row.id) for row in self.rows if isinstance(row,tuple) and row.id in data} @@ -717,7 +724,7 @@ class Ticker: self.upd_w = max_w def init_prec(self): - exp = [(a.id,Decimal.adjusted(self.prices[a.id]['usd-us-dollar'])) for a in self.usr_col_assets] + exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted() ) for a in self.usr_col_assets] self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp } self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp } @@ -737,13 +744,13 @@ class Ticker: for asset in self.usr_col_assets: if asset.symbol != 'USD': - usdprice = Decimal(self.data[asset.id]['price_usd']) + usdprice = self.data[asset.id]['price_usd'] yield '{} ({}) = {:{}.{}f} USD'.format( asset.symbol, self.create_label(asset.id), usdprice, self.comma, - max(2,int(-usdprice.adjusted())+4) ) + max(2, 4-usdprice.adjusted()) ) if hasattr(self,'subhdr'): yield self.subhdr @@ -818,17 +825,14 @@ class Ticker: if id in self.data: d = self.data[id] return { k: ( - Decimal(d['price_btc']) if k == 'btc-bitcoin' else - Decimal(d['price_usd']) / self.col_usd_prices[k] + d['price_btc'] if k == 'btc-bitcoin' else + d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids } def fmt_row(self,d,amt=None,amt_fmt=None): - def fmt_pct(d): - if d in ('',None): - return gray(' --') - n = Decimal(d) - return (red,green)[n>=0](f'{n:+7.2f}') + def fmt_pct(n): + return gray(' --') if n == None else (red,green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] @@ -927,8 +931,8 @@ class Ticker: if self.offer: real_price = ( self.asset.amount - * Decimal(data[self.asset.id]['price_usd']) - / Decimal(data[self.to_asset.id]['price_usd']) + * data[self.asset.id]['price_usd'] + / data[self.to_asset.id]['price_usd'] ) if self.adjust != 1: die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier') @@ -943,7 +947,7 @@ class Ticker: self.usr_col_assets = [self.asset] + ([self.to_asset] if self.to_asset else []) for a in self.usr_col_assets: - self.prices[a.id]['usd-us-dollar'] = Decimal(data[a.id]['price_usd']) + self.prices[a.id]['usd-us-dollar'] = data[a.id]['price_usd'] self.format_last_update_col(cross_assets=self.usr_col_assets) @@ -953,7 +957,7 @@ class Ticker: def get_row_prices(self,id): if id in self.data: d = self.data[id] - return { k: self.col_usd_prices[self.asset.id] / Decimal(d['price_usd']) for k in self.col_ids } + return { k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids } def init_fs(self): self.max_wid = max( From 77d8ffb7b9accfa086a1b8e89f494b393c2940de Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 037/108] mmnode-ticker: use coinpaprika `tickers` API call --- mmgen_node_tools/Ticker.py | 28 +++++++++++++++++----------- mmgen_node_tools/main_ticker.py | 20 +++++++++++++------- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 12 ++++++------ test/ref/ticker/ticker-btc.json | 2 +- test/ref/ticker/ticker.json | 2 +- 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index b2fc86a..40bede8 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -12,14 +12,14 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets """ -# We use deprecated coinpaprika ‘ticker’ API for now because it returns ~45% less data. +# v3.2.dev4: switch to new coinpaprika ‘tickers’ API call (supports ‘limit’ parameter, more historical data) # Old ‘ticker’ API (/v1/ticker): data['BTC']['price_usd'] # New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price'] # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,re,time,json,yaml,random +import sys,os,re,time,datetime,json,yaml,random from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple @@ -159,6 +159,10 @@ class DataSource: btc_ratelimit = 10 net_data_type = 'json' has_verbose = True + dfl_asset_limit = 2000 + + def __init__(self): + self.asset_limit = int(gcfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self,elapsed): return ( @@ -168,7 +172,10 @@ class DataSource: @property def api_url(self): - return f'https://{self.api_host}/v1/ticker' + ('/btc-bitcoin' if cfg.btc_only else '') + return ( + f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else + f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else + f'https://{self.api_host}/v1/tickers' ) @property def json_fn(self): @@ -352,7 +359,7 @@ def gen_data(data): for d in data['cc']: if d['id'] == 'btc-bitcoin': - btcusd = Decimal(d['price_usd']) + btcusd = Decimal(str(d['quotes']['USD']['price'])) break get_id = src_cls['fi'].get_id @@ -378,13 +385,12 @@ def gen_data(data): if d[k] in wants[k]: if d[k] in found[k]: die(1,dup_sym_errmsg(d[k])) - if not d.get('_converted'): - d['price_usd'] = Decimal(d['price_usd']) - d['price_btc'] = Decimal(d['price_btc']) - d['percent_change_7d'] = Decimal(d['percent_change_7d']) - d['percent_change_24h'] = Decimal(d['percent_change_24h']) - d['last_updated'] = int(d['last_updated']) - d['_converted'] = True + if not 'price_usd' in d: + d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) + d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd + d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] + d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp()) yield (d['id'],d) found[k].add(d[k]) wants[k].remove(d[k]) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 13175b8..91bc293 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -25,6 +25,9 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) +-a, --asset-limit=N Retrieve data for top ‘N’ cryptocurrencies by market + cap (default: {al}). To retrieve all available data, + specify a value of zero. -A, --adjust=P Adjust prices by percentage ‘P’. In ‘trading’ mode, spot and adjusted prices are shown in separate columns. -b, --btc Fetch and display data for Bitcoin only @@ -137,13 +140,14 @@ Financial data is obtained from {fi.desc}, which currently allows Tor. RATE LIMITING NOTE -To protect user privacy, all filtering and processing of cryptocurrency data -is performed client side so that the remote server does not know which assets -are being examined. This means that data for ALL available crypto assets -(currently over 8000) is fetched with each invocation of the script. A rate -limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the -remote server. When the --btc option is in effect, this limit is reduced to -{cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data. +To protect user privacy, filtering and processing of cryptocurrency data is +performed client side so that the remote server does not know which assets +are being examined. This is done by fetching data for the top {al} crypto +assets by market cap (configurable via the --asset-limit option) with each +invocation of the script. A rate limit of {cc.ratelimit} seconds between calls is thus +imposed to prevent abuse of the remote server. When the --btc option is in +effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the rate limit +entirely, use --cached-data. Note that financial data obtained from {fi.api_host} is filtered in the request, which has privacy implications. The rate limit for financial data @@ -201,11 +205,13 @@ To add a portfolio, edit the file 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), + al = DataSource.coinpaprika.dfl_asset_limit, ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), fi = src_cls['fi'](), ) diff --git a/setup.cfg b/setup.cfg index 55c315a..0c1e362 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ classifiers = Operating System :: Microsoft :: Windows [options] -python_requires = >=3.7 +python_requires = >=3.8 include_package_data = True install_requires = diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 966fb91..3a9c47c 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -132,7 +132,7 @@ class CmdTestScripts(CmdTestBase): [], [ 'USD BTC', - 'BTC 23250.77 1.00000000 ETH 1659.66 0.07146397' + 'BTC 23250.77 1.00000000 ETH 1659.66 0.07138094' ]) @@ -144,8 +144,8 @@ class CmdTestScripts(CmdTestBase): r'INR \(INDIAN RUPEE\) = 0.012579 USD', 'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED', 'BITCOIN', - r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07146397 \+21.42 \+1.82', - r'MONERO 158.97 149.3870 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', + r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+21.42 \+1.82', + r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+7.28 \+1.21 2022-08-02 18:25:59', r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --', ]) @@ -156,8 +156,8 @@ class CmdTestScripts(CmdTestBase): [ 'Adjusting prices by -0.77%', 'USD BTC CHG_7d CHG_24h UPDATED', - r'LITECOIN 58.56 0.00252162 \+12.79 \+0.40 2022-08-02 18:25:59', - r'MONERO 157.76 0.00679284 \+7.28 \+1.21' + r'LITECOIN 58.56 0.00251869 \+12.79 \+0.40 2022-08-02 18:25:59', + r'MONERO 157.76 0.00678495 \+7.28 \+1.21' ]) os.unlink(os.path.join(self.nt_datadir,'ticker-cfg.yaml')) return t @@ -174,7 +174,7 @@ class CmdTestScripts(CmdTestBase): ['--wide','--portfolio'], [ 'USD BTC CHG_7d CHG_24h UPDATED', - r'ETHEREUM 1,659.66 0.07146397 \+21.42 \+1.82 2022-08-02 18:25:59', + r'ETHEREUM 1,659.66 0.07138094 \+21.42 \+1.82 2022-08-02 18:25:59', 'CARDANO','ALGORAND', 'PORTFOLIO','BITCOIN','ETHEREUM','MONERO','CARDANO','ALGORAND','TOTAL' ]) diff --git a/test/ref/ticker/ticker-btc.json b/test/ref/ticker/ticker-btc.json index 75c8909..d0549b1 100644 --- a/test/ref/ticker/ticker-btc.json +++ b/test/ref/ticker/ticker-btc.json @@ -1 +1 @@ -{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23368.859557988893","price_btc":"1","volume_24h_usd":"24116251608.791744","market_cap_usd":"446560795287","circulating_supply":"19109225","total_supply":"19109231","max_supply":"21000000","percent_change_1h":"-0.23","percent_change_24h":"-1.87","percent_change_7d":"6.05","last_updated":"1659346445"} \ No newline at end of file +{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","circulating_supply":"19109225","total_supply":"19109231","max_supply":"21000000","last_updated":"2022-08-01T09:34:05Z","quotes":{"USD":{"price":23368.859557988893,"percent_change_1h":-0.23,"percent_change_6h":-1.4960000000000002,"percent_change_24h":-1.87,"percent_change_7d":6.05,"percent_change_30d":8.469999999999999,"percent_change_1y":10.285,"volume_24h":24116251608.791744,"market_cap":446560795287}}} diff --git a/test/ref/ticker/ticker.json b/test/ref/ticker/ticker.json index 880a666..4535b12 100644 --- a/test/ref/ticker/ticker.json +++ b/test/ref/ticker/ticker.json @@ -1 +1 @@ -[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"}] +[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":23250.774053363122,"percent_change_1h":-0.27,"percent_change_6h":0.7120000000000001,"percent_change_24h":0.89,"percent_change_7d":11.15,"percent_change_30d":15.61,"percent_change_1y":18.955000000000002,"volume_24h":28307579008.4866,"market_cap":444336521633}}},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":1659.6621665887371,"percent_change_1h":-0.03,"percent_change_6h":1.4560000000000002,"percent_change_24h":1.82,"percent_change_7d":21.42,"percent_change_30d":29.988,"percent_change_1y":36.414,"volume_24h":18216561308.363518,"market_cap":202151289827}}},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":59.013589192027936,"percent_change_1h":-0.43,"percent_change_6h":0.32000000000000006,"percent_change_24h":0.4,"percent_change_7d":12.79,"percent_change_30d":17.906,"percent_change_1y":21.743,"volume_24h":510336800.72056556,"market_cap":4181502638}}},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":158.97302629813817,"percent_change_1h":0.21,"percent_change_6h":0.968,"percent_change_24h":1.21,"percent_change_7d":7.28,"percent_change_30d":10.192,"percent_change_1y":12.376,"volume_24h":78159392.30003875,"market_cap":2886277702}}},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":0.5069722536476557,"percent_change_1h":0.05,"percent_change_6h":-0.08800000000000001,"percent_change_24h":-0.11,"percent_change_7d":11.68,"percent_change_30d":16.352,"percent_change_1y":19.855999999999998,"volume_24h":507973898.04662097,"market_cap":17111613980}}},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":0.33196893931152616,"percent_change_1h":-0.06,"percent_change_6h":-0.656,"percent_change_24h":-0.82,"percent_change_7d":9.69,"percent_change_30d":13.565999999999999,"percent_change_1y":16.473,"volume_24h":62207779.35759487,"market_cap":2306909784}}}] From 545bc044c6e89d48e814cb64e8649ffbba637d85 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 038/108] mmnode-ticker: add year and month percentage change columns --- mmgen_node_tools/Ticker.py | 42 +++++++++++++++++++++++++++------ mmgen_node_tools/main_ticker.py | 21 ++++++++++++----- test/cmdtest_py_d/ct_misc.py | 11 +++++---- 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 40bede8..4011c90 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -34,6 +34,13 @@ portfolio_fn = 'ticker-portfolio.yaml' asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) last_api_host = None +percent_cols = { + 'd': 'day', + 'w': 'week', + 'm': 'month', + 'y': 'year', +} + class DataSource: source_groups = [ @@ -390,6 +397,8 @@ def gen_data(data): d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] + d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp()) yield (d['id'],d) found[k].add(d[k]) @@ -497,6 +506,15 @@ def make_cfg(gcfg_arg): yield parse_asset_id(e,require_label=True) return tuple(gen()) + def parse_percent_cols(arg): + if arg is None: + return [] + res = arg.lower().split(',') + for s in res: + if s not in percent_cols: + die(1,f'{arg!r}: invalid --percent-cols parameter (valid letters: {fmt_list(percent_cols)})') + return res + def parse_usr_asset_arg(key,use_cf_file=False): """ asset_id[:rate[:rate_asset]] @@ -606,7 +624,8 @@ def make_cfg(gcfg_arg): 'cachedir', 'proxy', 'proxy2', - 'portfolio' ]) + 'portfolio', + 'percent_cols' ]) global gcfg,cfg_in,src_cls,cfg @@ -645,7 +664,8 @@ def make_cfg(gcfg_arg): cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None + portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None, + percent_cols = parse_percent_cols(gcfg.percent_cols) ) def get_cfg_in(): @@ -802,7 +822,7 @@ class Ticker: yield '-' * self.hl_wid if not cfg.btc_only: yield self.fs_num.format( - lbl = 'TOTAL', pc1='', pc2='', upd='', amt='', + lbl = 'TOTAL', pc3='', pc4='', pc1='', pc2='', upd='', amt='', **{ k.replace('-','_'): v for k,v in self.prices['total'].items() } ) @@ -851,6 +871,8 @@ class Ticker: lbl = (self.create_label(d['id']) if gcfg.name_labels else d['symbol']), pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), + pc3 = fmt_pct(d.get('percent_change_1y')), + pc4 = fmt_pct(d.get('percent_change_30d')), upd = d.get('last_updated_fmt'), amt = amt_fmt, **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() } @@ -873,8 +895,10 @@ class Ticker: col_fs_data = { 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid), - 'pct7d': fd(' {pc1:7}', ' {pc1:7}', 8), - 'pct24h': fd(' {pc2:7}', ' {pc2:7}', 8), + 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), + 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), + 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), + 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2), 'amt': fd(' {amt}', ' {amt}', 21), } @@ -891,8 +915,10 @@ class Ticker: [asset.id for asset in self.usr_col_assets] + [a for a,b in ( ( 'btc-bitcoin', not cfg.btc_only ), - ( 'pct7d', gcfg.percent_change ), - ( 'pct24h', gcfg.percent_change ), + ( 'pct1y', 'y' in cfg.percent_cols ), + ( 'pct1m', 'm' in cfg.percent_cols ), + ( 'pct1w', 'w' in cfg.percent_cols ), + ( 'pct1d', 'd' in cfg.percent_cols ), ( 'update_time', gcfg.update_time ), ) if b] ) @@ -915,6 +941,8 @@ class Ticker: lbl = '', pc1 = ' CHG_7d', pc2 = 'CHG_24h', + pc3 = 'CHG_1y', + pc4 = 'CHG_30d', upd = 'UPDATED', amt = ' AMOUNT', usd_us_dollar = 'USD', diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 91bc293..eafa34a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -14,7 +14,11 @@ mmnode-ticker: Display price information for cryptocurrency and other assets opts_data = { 'sets': [ - ('wide', True, 'percent_change', True), + ('widest', True, 'percent_cols', 'd,w,m,y'), + ('widest', True, 'name_labels', True), + ('widest', True, 'thousands_comma', True), + ('widest', True, 'update_time', True), + ('wide', True, 'percent_cols', 'd,w'), ('wide', True, 'name_labels', True), ('wide', True, 'thousands_comma', True), ('wide', True, 'update_time', True), @@ -45,7 +49,9 @@ opts_data = { -F, --portfolio Display portfolio data -l, --list-ids List IDs of all available assets -n, --name-labels Label rows with asset names rather than symbols --p, --percent-change Add percentage change columns +-p, --percent-cols=C Add daily, weekly, monthly, or yearly percentage change + columns ‘C’ (specify with comma-separated letters + {pc}) -P, --pager Pipe the output to a pager -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply @@ -54,7 +60,8 @@ opts_data = { -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column -v, --verbose Be more verbose --w, --wide Display all optional columns (equivalent to -punT) +-w, --wide Display most optional columns (same as -unT -p d,w) +-W, --widest Display all optional columns (same as -unT -p d,w,m,y) -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to completely disable or ‘none’ to allow override from environment. Consult the curl manpage for --proxy usage. @@ -169,9 +176,10 @@ $ mmnode-ticker -w -c eurusd=x,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 # Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate: $ mmnode-ticker -wE -c eurusd=x,bgn-bulgarian-lev:0.5113r:eurusd=x -# Wide display, use cached data from previous network query, show portfolio -# (see above), pipe output to pager, add DOGE row: -$ mmnode-ticker -wCFP -r doge +# Widest display with all percentage change columns, use cached data from +# previous network query, show portfolio (see above), pipe output to pager, +# add DOGE row: +$ mmnode-ticker -WCFP -r doge # Display 17.234 XMR priced in all configured assets (‘trading’ mode): $ mmnode-ticker xmr:17.234 @@ -206,6 +214,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, + pc = fmt_list(Ticker.percent_cols,fmt='bare'), ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 3a9c47c..8046e51 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -138,15 +138,16 @@ class CmdTestScripts(CmdTestBase): def ticker4(self): return self.ticker( - ['--wide','--add-columns=eurusd=x,inr-indian-rupee:79.5'], + ['--widest','--add-columns=eurusd=x,inr-indian-rupee:79.5'], [ r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' + r'INR \(INDIAN RUPEE\) = 0.012579 USD', - 'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED', + 'USD EURUSD=X INR BTC CHG_1y CHG_30d CHG_7d CHG_24h UPDATED', 'BITCOIN', - r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+21.42 \+1.82', - r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+7.28 \+1.21 2022-08-02 18:25:59', - r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --', + r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+36.41 \+29.99 \+21.42 \+1.82', + r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+12.38 \+10.19 \+7.28 \+1.21 2022-08-02 18:25:59', + r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -- -- -- -0.23', + r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- -- -- --', ]) def ticker5(self): From af7c14fe369fc37d4d465e9a4bcb9f35b1952173 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 039/108] mmnode-ticker: get historical financial data from Yahoo Finance --- mmgen_node_tools/Ticker.py | 43 ++++++++++++++++++++- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 3 +- test/ref/ticker/ticker-finance-history.json | 1 + 5 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 test/ref/ticker/ticker-finance-history.json diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 4011c90..6c15906 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -48,6 +48,7 @@ class DataSource: 'cc': 'coinpaprika' }, { 'fi': 'yahoospot', + 'hi': 'yahoohist', } ] @@ -248,7 +249,9 @@ class DataSource: 'symbol': sym.upper(), 'price_usd': price_usd, 'price_btc': price_usd / btcusd, - 'percent_change_7d': None, + 'percent_change_1y': data['pct_chg_1y'], + 'percent_change_30d': data['pct_chg_4wks'], + 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, 'last_updated': data['regularMarketTime'], } @@ -272,6 +275,7 @@ class DataSource: kwargs = { 'formatted': True, + 'asynchronous': True, 'proxies': { 'https': cfg.proxy2 }, } @@ -297,6 +301,33 @@ class DataSource: id = s.lower(), source = 'fi' ) + class yahoohist(yahoospot): + + json_fn_basename = 'ticker-finance-history.json' + data_desc = 'historical financial data' + net_data_type = 'json' + period = '1y' + interval = '1wk' + + def process_network_data(self,ticker): + return ticker.history( + period = self.period, + interval = self.interval).to_json(orient='index') + + def postprocess_data(self,data): + def gen(): + keys = set() + for key,val in data.items(): + if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): + date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) + if (sym := m[1]) in keys: + d[date] = val + else: + keys.add(sym) + d = {date:val} + yield (sym,d) + return dict(gen()) + def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield '' @@ -378,6 +409,16 @@ def gen_data(data): if id in wants['id']: if id in found['id']: die(1,dup_sym_errmsg(id)) + if m := data['hi'].get(k): + spot = v['regularMarketPrice']['raw'] + hist = tuple(m.values()) + v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( + (spot / hist[-2]['close'] - 1) * 100, + (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month + (spot / hist[0]['close'] - 1) * 100, + ) + else: + v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None yield ( id, conv_func(id,v,btcusd) ) found['id'].add(id) wants['id'].remove(id) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 5223fbd..ce7b703 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev3 +3.2.dev4 diff --git a/setup.cfg b/setup.cfg index 0c1e362..339e7f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen>=14.0.dev9 + mmgen>=14.0.dev11 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 8046e51..3dbac50 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -102,6 +102,7 @@ class CmdTestScripts(CmdTestBase): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) + shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' @@ -146,7 +147,7 @@ class CmdTestScripts(CmdTestBase): 'BITCOIN', r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+36.41 \+29.99 \+21.42 \+1.82', r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+12.38 \+10.19 \+7.28 \+1.21 2022-08-02 18:25:59', - r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -- -- -- -0.23', + r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -1.71 \+12.93 \+9.05 -0.23', r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- -- -- --', ]) diff --git a/test/ref/ticker/ticker-finance-history.json b/test/ref/ticker/ticker-finance-history.json new file mode 100644 index 0000000..e8b0bd3 --- /dev/null +++ b/test/ref/ticker/ticker-finance-history.json @@ -0,0 +1 @@ +{"('BZ=F', datetime.date(2021, 8, 2))":{"open":75.1800003052,"high":75.3300018311,"low":69.75,"close":70.6999969482,"volume":192386,"adjclose":70.6999969482},"('BZ=F', datetime.date(2022, 6, 27))":{"open":112.4599990845,"high":120.3799972534,"low":108.0100021362,"close":111.6299972534,"volume":64094,"adjclose":111.6299972534},"('BZ=F', datetime.date(2022, 7, 4))":{"open":111.6100006104,"high":114.6900024414,"low":98.4700012207,"close":107.0199966431,"volume":115623,"adjclose":107.0199966431},"('BZ=F', datetime.date(2022, 7, 11))":{"open":106.7600021362,"high":107.6600036621,"low":94.5,"close":101.1600036621,"volume":109537,"adjclose":101.1600036621},"('BZ=F', datetime.date(2022, 7, 18))":{"open":100.9199981689,"high":107.5999984741,"low":99.4800033569,"close":103.1999969482,"volume":110290,"adjclose":103.1999969482},"('BZ=F', datetime.date(2022, 7, 25))":{"open":103.4400024414,"high":110.4300003052,"low":101.6699981689,"close":110.0100021362,"volume":77572,"adjclose":110.0100021362},"('BZ=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":102.4400024414,"high":111.4300003052,"low":101.6699981689,"close":111.0100021362,"volume":77572,"adjclose":111.0100021362},"('CHFUSD=X', datetime.date(2021, 7, 26))":{"open":1.0939244032,"high":1.1064150333,"low":1.0908932686,"close":1.1047636271,"volume":0,"adjclose":1.1047636271},"('CHFUSD=X', datetime.date(2022, 7, 4))":{"open":1.043405652,"high":1.0446264744,"low":1.020783186,"close":1.0241703987,"volume":0,"adjclose":1.0241703987},"('CHFUSD=X', datetime.date(2022, 7, 11))":{"open":1.022777319,"high":1.0247056484,"low":1.0118077993,"close":1.0238809586,"volume":0,"adjclose":1.0238809586},"('CHFUSD=X', datetime.date(2022, 7, 18))":{"open":1.0250102282,"high":1.0417101383,"low":1.0216904879,"close":1.0396090746,"volume":0,"adjclose":1.0396090746},"('CHFUSD=X', datetime.date(2022, 7, 25))":{"open":1.0384216309,"high":1.0522992611,"low":1.0345113277,"close":1.0506275892,"volume":0,"adjclose":1.0506275892},"('CHFUSD=X', datetime.date(2022, 8, 1))":{"open":1.0492409468,"high":1.0547411442,"low":1.0495601892,"close":1.0492409468,"volume":0,"adjclose":1.0492409468},"('CHFUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.0492409468,"high":1.0547411442,"low":1.0495601892,"close":1.0492409468,"volume":0,"adjclose":1.0492409468},"('EURUSD=X', datetime.date(2021, 7, 26))":{"open":1.1822398901,"high":1.1911001205,"low":1.1785781384,"close":1.1867604256,"volume":0,"adjclose":1.1867604256},"('EURUSD=X', datetime.date(2022, 7, 4))":{"open":1.043394804,"high":1.0463534594,"low":1.0079730749,"close":1.0187449455,"volume":0,"adjclose":1.0187449455},"('EURUSD=X', datetime.date(2022, 7, 11))":{"open":1.0166114569,"high":1.0167768002,"low":0.9953616261,"close":1.008867979,"volume":0,"adjclose":1.008867979},"('EURUSD=X', datetime.date(2022, 7, 18))":{"open":1.0096318722,"high":1.0275379419,"low":1.0082373619,"close":1.0215548277,"volume":0,"adjclose":1.0215548277},"('EURUSD=X', datetime.date(2022, 7, 25))":{"open":1.0200231075,"high":1.0256409645,"low":1.0107442141,"close":1.0227040052,"volume":0,"adjclose":1.0227040052},"('EURUSD=X', datetime.date(2022, 8, 1))":{"open":1.02082479,"high":1.0277491808,"low":1.020960331,"close":1.02082479,"volume":0,"adjclose":1.02082479},"('EURUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.02082479,"high":1.0277491808,"low":1.020960331,"close":1.02082479,"volume":0,"adjclose":1.02082479},"('GBPUSD=X', datetime.date(2021, 7, 26))":{"open":1.3886767626,"high":1.3984057903,"low":1.3846387863,"close":1.3910005093,"volume":0,"adjclose":1.3910005093},"('GBPUSD=X', datetime.date(2022, 7, 4))":{"open":1.2105804682,"high":1.2165154219,"low":1.1877613068,"close":1.2031002045,"volume":0,"adjclose":1.2031002045},"('GBPUSD=X', datetime.date(2022, 7, 11))":{"open":1.2017786503,"high":1.2019952536,"low":1.1763184071,"close":1.1855996847,"volume":0,"adjclose":1.1855996847},"('GBPUSD=X', datetime.date(2022, 7, 18))":{"open":1.1887921095,"high":1.2064181566,"low":1.1875497103,"close":1.2005999088,"volume":0,"adjclose":1.2005999088},"('GBPUSD=X', datetime.date(2022, 7, 25))":{"open":1.1984229088,"high":1.2242598534,"low":1.1962151527,"close":1.2180001736,"volume":0,"adjclose":1.2180001736},"('GBPUSD=X', datetime.date(2022, 8, 1))":{"open":1.2167818546,"high":1.2291958332,"low":1.2164857388,"close":1.2167373896,"volume":0,"adjclose":1.2167373896},"('GBPUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.2167818546,"high":1.2291958332,"low":1.2164857388,"close":1.2167373896,"volume":0,"adjclose":1.2167373896},"('GC=F', datetime.date(2021, 7, 26))":{"open":1803.6999511719,"high":1832.5999755859,"low":1799.5,"close":1812.5999755859,"volume":243380,"adjclose":1812.5999755859},"('GC=F', datetime.date(2022, 6, 27))":{"open":1830.5,"high":1830.6999511719,"low":1791.5999755859,"close":1798.9000244141,"volume":1046,"adjclose":1798.9000244141},"('GC=F', datetime.date(2022, 7, 4))":{"open":1805.4000244141,"high":1805.4000244141,"low":1732.0999755859,"close":1740.5999755859,"volume":4032,"adjclose":1740.5999755859},"('GC=F', datetime.date(2022, 7, 11))":{"open":1732.5,"high":1736.6999511719,"low":1701.0999755859,"close":1702.4000244141,"volume":2904,"adjclose":1702.4000244141},"('GC=F', datetime.date(2022, 7, 18))":{"open":1712.1999511719,"high":1735,"low":1679.8000488281,"close":1727.0999755859,"volume":2322,"adjclose":1727.0999755859},"('GC=F', datetime.date(2022, 7, 25))":{"open":1727,"high":1765.6999511719,"low":1717.6999511719,"close":1762.9000244141,"volume":185654,"adjclose":1762.9000244141},"('GC=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1727,"high":1765.6999511719,"low":1717.6999511719,"close":1762.9000244141,"volume":185654,"adjclose":1762.9000244141},"('SI=F', datetime.date(2021, 7, 26))":{"open":25.2000007629,"high":25.8549995422,"low":24.6319999695,"close":25.5279998779,"volume":1000,"adjclose":25.5279998779},"('SI=F', datetime.date(2022, 6, 27))":{"open":21.2950000763,"high":21.3299999237,"low":19.2649993896,"close":19.5970001221,"volume":48988,"adjclose":19.5970001221},"('SI=F', datetime.date(2022, 7, 4))":{"open":19.8099994659,"high":20.0699996948,"low":18.8400001526,"close":19.1669998169,"volume":566,"adjclose":19.1669998169},"('SI=F', datetime.date(2022, 7, 11))":{"open":19.1650009155,"high":19.1749992371,"low":18,"close":18.5480003357,"volume":1108,"adjclose":18.5480003357},"('SI=F', datetime.date(2022, 7, 18))":{"open":18.8099994659,"high":18.9400005341,"low":18.1100006104,"close":18.5849990845,"volume":690,"adjclose":18.5849990845},"('SI=F', datetime.date(2022, 7, 25))":{"open":18.4099998474,"high":20.2900009155,"low":18.1849994659,"close":20.1560001373,"volume":980,"adjclose":20.1560001373},"('SI=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":18.4099998474,"high":20.2900009155,"low":18.1849994659,"close":20.1560001373,"volume":980,"adjclose":20.1560001373},"('^DJI', datetime.date(2021, 7, 26))":{"open":35078.8984375,"high":35171.51953125,"low":34871.12890625,"close":34935.46875,"volume":1172870000,"adjclose":34935.46875},"('^DJI', datetime.date(2022, 6, 27))":{"open":31533.599609375,"high":31885.08984375,"low":30431.869140625,"close":31097.259765625,"volume":1634590000,"adjclose":31097.259765625},"('^DJI', datetime.date(2022, 7, 4))":{"open":30903.119140625,"high":31511.4609375,"low":30355.119140625,"close":31338.150390625,"volume":1147620000,"adjclose":31338.150390625},"('^DJI', datetime.date(2022, 7, 11))":{"open":31277.98046875,"high":31367.55078125,"low":30143.9296875,"close":31288.259765625,"volume":1507740000,"adjclose":31288.259765625},"('^DJI', datetime.date(2022, 7, 18))":{"open":31475.98046875,"high":32219.25,"low":30982.970703125,"close":31899.2890625,"volume":1624400000,"adjclose":31899.2890625},"('^DJI', datetime.date(2022, 7, 25))":{"open":31950.9296875,"high":32910.1796875,"low":31705.359375,"close":32845.12890625,"volume":1764330000,"adjclose":32845.12890625},"('^DJI', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":31950.9296875,"high":32910.1796875,"low":31705.359375,"close":32845.12890625,"volume":1764330000,"adjclose":32845.12890625},"('^GSPC', datetime.date(2021, 7, 26))":{"open":4416.3798828125,"high":4429.9702148438,"low":4372.509765625,"close":4395.259765625,"volume":16458580000,"adjclose":4395.259765625},"('^GSPC', datetime.date(2022, 6, 27))":{"open":3920.7600097656,"high":3945.8601074219,"low":3738.669921875,"close":3825.330078125,"volume":21693690000,"adjclose":3825.330078125},"('^GSPC', datetime.date(2022, 7, 4))":{"open":3792.6101074219,"high":3918.5,"low":3742.0600585938,"close":3899.3798828125,"volume":17073700000,"adjclose":3899.3798828125},"('^GSPC', datetime.date(2022, 7, 11))":{"open":3880.9399414062,"high":3880.9399414062,"low":3721.5600585938,"close":3863.1599121094,"volume":19693570000,"adjclose":3863.1599121094},"('^GSPC', datetime.date(2022, 7, 18))":{"open":3883.7900390625,"high":4012.4399414062,"low":3818.6298828125,"close":3961.6298828125,"volume":20385270000,"adjclose":3961.6298828125},"('^GSPC', datetime.date(2022, 7, 25))":{"open":3965.7199707031,"high":4140.1499023438,"low":3910.7399902344,"close":4130.2900390625,"volume":20488830000,"adjclose":4130.2900390625},"('^GSPC', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":3965.7199707031,"high":4140.1499023438,"low":3910.7399902344,"close":4130.2900390625,"volume":20488830000,"adjclose":4130.2900390625},"('^IXIC', datetime.date(2021, 7, 26))":{"open":14807.9501953125,"high":14833.740234375,"low":14503.759765625,"close":14672.6796875,"volume":16171130000,"adjclose":14672.6796875},"('^IXIC', datetime.date(2022, 6, 27))":{"open":11661.01953125,"high":11677.490234375,"low":10850.009765625,"close":11127.849609375,"volume":26652190000,"adjclose":11127.849609375},"('^IXIC', datetime.date(2022, 7, 4))":{"open":10964.1796875,"high":11689.7001953125,"low":10911.4501953125,"close":11635.3095703125,"volume":19116360000,"adjclose":11635.3095703125},"('^IXIC', datetime.date(2022, 7, 11))":{"open":11524.490234375,"high":11541.099609375,"low":11005.9296875,"close":11452.419921875,"volume":21963960000,"adjclose":11452.419921875},"('^IXIC', datetime.date(2022, 7, 18))":{"open":11561.6396484375,"high":12093.01953125,"low":11322.83984375,"close":11834.1103515625,"volume":25230550000,"adjclose":11834.1103515625},"('^IXIC', datetime.date(2022, 7, 25))":{"open":11837.9599609375,"high":12426.259765625,"low":11533.3701171875,"close":12390.6904296875,"volume":23117120000,"adjclose":12390.6904296875},"('^IXIC', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":11837.9599609375,"high":12426.259765625,"low":11533.3701171875,"close":12390.6904296875,"volume":23117120000,"adjclose":12390.6904296875}} From 1215085fbd78e89bc3786c6af29f996a5fa4b220 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 8 Nov 2023 08:02:05 +0000 Subject: [PATCH 040/108] setup.cfg: update dependency to `mmgen-wallet` --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index ce7b703..67288ca 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev4 +3.2.dev5 diff --git a/setup.cfg b/setup.cfg index 339e7f7..3cb1afb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen>=14.0.dev11 + mmgen-wallet>=14.0.dev12 pyyaml yahooquery From b2dc247429927471aa3d402894c2bd692a027684 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 8 Nov 2023 10:31:42 +0000 Subject: [PATCH 041/108] update package name, install instructions in README --- README.md | 8 +++++++- setup.cfg | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 673155e..96807ad 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,13 @@ Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. ## Install: -First, install [MMGen][6]. +### Stable version: + + $ python3 -m pip install --user --upgrade mmgen-node-tools + +### Development version: + +First, install the latest development version of [MMGen][6]. Then, diff --git a/setup.cfg b/setup.cfg index 3cb1afb..ae7676f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = MMGen Node Tools +name = mmgen-node-tools version = file: mmgen_node_tools/data/version description = Optional online tools for the MMGen wallet suite long_description = file: README.md From 554f78690384cf297a9770036a87a701d9e55766 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 9 Nov 2023 05:19:01 +0000 Subject: [PATCH 042/108] update install instructions in README.md --- README.md | 13 +++++++++---- mmgen_node_tools/data/version | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 96807ad..c7ecc29 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,15 @@ Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. ## Install: +If installing as user (without venv), Make sure that `~/.local/bin` is in `PATH`. + ### Stable version: - $ python3 -m pip install --user --upgrade mmgen-node-tools +First, install the [required MMGen packages][7] for your Linux distribution: + +Then, + + $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: @@ -21,9 +27,7 @@ Then, $ git clone https://github.com/mmgen/mmgen-node-tools $ cd mmgen-node-tools $ python3 -m build --no-isolation - $ python3 -m pip install --user dist/*.whl - -Also make sure that `~/.local/bin` is in `PATH`. + $ python3 -m pip install dist/*.whl ## Test: @@ -52,3 +56,4 @@ Donate: 15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w [4]: https://bitcointalk.org/index.php?topic=567069.0 [5]: https://github.com/mmgen/mmgen/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen/ +[7]: https://github.com/mmgen/mmgen/wiki/Install-MMGen-on-Linux diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 67288ca..5d6bce8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev5 +3.2.dev6 From 56327412ca910fd44ba278e9292c31c8160dda36 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Nov 2023 14:45:15 +0000 Subject: [PATCH 043/108] update for mmgen-wallet v14.0.dev22 --- README.md | 14 +++++++------- mmgen_node_tools/Misc.py | 4 ++-- mmgen_node_tools/PeerBlocks.py | 4 ++-- mmgen_node_tools/PollDisplay.py | 4 ++-- mmgen_node_tools/Ticker.py | 4 ++-- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 4 ++-- mmgen_node_tools/main_ticker.py | 4 ++-- setup.cfg | 7 ++++++- test/cmdtest_py_d/ct_main.py | 4 ++-- test/init.sh | 12 ++++++------ .../fakemods/mmgen_node_tools/PeerBlocks.py | 4 ++-- 12 files changed, 36 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index c7ecc29..c95e206 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. ## Install: -If installing as user (without venv), Make sure that `~/.local/bin` is in `PATH`. +If installing as user (without venv), make sure that `~/.local/bin` is in `PATH`. ### Stable version: -First, install the [required MMGen packages][7] for your Linux distribution: +First, install the [required MMGen Wallet packages][7] for your Linux distribution: Then, @@ -20,7 +20,7 @@ Then, ### Development version: -First, install the latest development version of [MMGen][6]. +First, install the latest development version of [MMGen Wallet][6]. Then, @@ -31,7 +31,7 @@ Then, ## Test: -*NOTE: the tests require that the MMGen and MMGen Node Tools repositories be +*NOTE: the tests require that the MMGen Wallet and MMGen Node Tools repositories be located in the same directory.* Initialize the test framework (must be run at least once after cloning, and @@ -54,6 +54,6 @@ Full testing: Donate: 15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w [4]: https://bitcointalk.org/index.php?topic=567069.0 -[5]: https://github.com/mmgen/mmgen/wiki/MMGen-Signing-Keys -[6]: https://github.com/mmgen/mmgen/ -[7]: https://github.com/mmgen/mmgen/wiki/Install-MMGen-on-Linux +[5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys +[6]: https://github.com/mmgen/mmgen-wallet/ +[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux diff --git a/mmgen_node_tools/Misc.py b/mmgen_node_tools/Misc.py index 999b68b..5aa27f5 100755 --- a/mmgen_node_tools/Misc.py +++ b/mmgen_node_tools/Misc.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmgen_node_tools.Misc: miscellaneous data and functions for the MMGen Node Tools suite diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index 0b386fb..c3d7c68 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 5bbfecd..72bedac 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ mmgen_node_tools.PollDisplay: update and display RPC data; get input from user diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6c15906..5a6a6a2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 5d6bce8..2c19032 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev6 +3.2.dev7 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 34c6236..7af2b77 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index eafa34a..f758089 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmnode-ticker: Display price information for cryptocurrency and other assets diff --git a/setup.cfg b/setup.cfg index ae7676f..4cb6a92 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,19 +11,24 @@ license = GNU GPL v3 platforms = Linux, Armbian, Raspbian, MS Windows keywords = file: mmgen_node_tools/data/keywords project_urls = + Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = Programming Language :: Python :: 3 License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows + Environment :: Console + Topic :: Office/Business :: Financial + Topic :: Security :: Cryptography + Development Status :: 5 - Production/Stable [options] python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.dev12 + mmgen-wallet>=14.0.dev22 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index ad1ce41..5f5c691 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite diff --git a/test/init.sh b/test/init.sh index 913ac9c..769490a 100755 --- a/test/init.sh +++ b/test/init.sh @@ -31,22 +31,22 @@ done shift $((OPTIND-1)) -mm_repo='../mmgen' +wallet_repo='../mmgen-wallet' die() { echo -e ${YELLOW}ERROR: $1$RESET; false; } becho() { echo -e $BLUE$1$RESET; } check_mmgen_repo() { - ( cd $mm_repo; python3 ./setup.py --url | grep -iq 'mmgen' ) + ( cd $wallet_repo; python3 ./setup.py --url | grep -iq 'mmgen' ) } build_mmgen_extmod() { - ( cd $mm_repo; python3 ./setup.py build_ext --inplace ) + ( cd $wallet_repo; python3 ./setup.py build_ext --inplace ) } create_dir_links() { for link_name in 'mmgen' 'scripts'; do - target="$mm_repo/$link_name" + target="$wallet_repo/$link_name" if [ -e $link_name ]; then [ $(realpath --relative-to=. $link_name) == $target ] || die "'$link_name' does not point to '$target'" else @@ -73,7 +73,7 @@ create_test_links() { [ "$path" ] || continue pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) - target="$mm_repo/$path" + target="$wallet_repo/$path" if [ ! -e "$target" ]; then echo "Target path $target is missing! Cannot proceed" exit 1 @@ -101,7 +101,7 @@ set -e becho 'Initializing MMGen Node Tools Test Suite' -check_mmgen_repo || die "MMGen repository not found at $mm_repo!" +check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py index 2484628..7c2fdf3 100644 --- a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ fakemods.mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes - test data From d0874f9a2d6475d81e50f6809aca6a05c4dced88 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 21 Nov 2023 15:30:42 +0000 Subject: [PATCH 044/108] test/init.sh: handle broken dir symlinks --- mmgen_node_tools/data/version | 2 +- test/cmdtest_py_d/cfg.py | 5 +++-- test/init.sh | 12 +++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 2c19032..21ca9e7 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev7 +3.2.dev8 diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_py_d/cfg.py index 5aacbf9..96bfe9f 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -12,7 +12,7 @@ test.cmdtest_py_d.cfg: configuration data for cmdtest.py """ -import os +cmd_groups_altcoin = [] cmd_groups_dfl = { 'main': ('CmdTestMain',{}), @@ -29,4 +29,5 @@ cfgs = { '3': {}, # main } -def fixup_cfgs(): pass +def fixup_cfgs(): + pass diff --git a/test/init.sh b/test/init.sh index 769490a..81727bb 100755 --- a/test/init.sh +++ b/test/init.sh @@ -47,9 +47,15 @@ build_mmgen_extmod() { create_dir_links() { for link_name in 'mmgen' 'scripts'; do target="$wallet_repo/$link_name" - if [ -e $link_name ]; then - [ $(realpath --relative-to=. $link_name) == $target ] || die "'$link_name' does not point to '$target'" - else + if [ -L $link_name ]; then + [ "$(realpath --relative-to=. $link_name 2>/dev/null)" == $target ] || { + echo "Removing broken symlink '$link_name'" + rm $link_name + } + elif [ -e $link_name ]; then + die "'$link_name' is not a symbolic link. Please remove or relocate it and re-run this script" + fi + if [ ! -e $link_name ]; then echo "Creating symlink: $link_name" ln -s $target fi From 55fdcb29f7ef476676cb8194dcdab690b08e3b46 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 27 Nov 2023 09:18:43 +0000 Subject: [PATCH 045/108] Support Windows/MSYS2 --- README.md | 82 ++++++++++++++++++++++++++------- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_main.py | 6 ++- test/cmdtest_py_d/ct_regtest.py | 4 +- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c95e206..d3b18ba 100644 --- a/README.md +++ b/README.md @@ -4,36 +4,66 @@ Requires modules from the [MMGen online/offline cryptocurrency wallet][6]. -Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. - ## Install: If installing as user (without venv), make sure that `~/.local/bin` is in `PATH`. +#### Windows/MSYS2: + +> Install [MSYS2 and the MMGen Wallet dependencies][8], skipping installation of +> scrypt, libsecp256k1 and the wallet itself if desired. + +> Install some additional dependencies: + +```bash + $ pacman -S \ + mingw-w64-ucrt-x86_64-python-pandas \ + mingw-w64-ucrt-x86_64-python-tqdm \ + mingw-w64-ucrt-x86_64-python-lxml + $ python3 -m pip install requests-futures + $ python3 -m pip install --no-deps yahooquery +``` + +#### Linux: + +> Install the [required MMGen Wallet packages][7] for your Linux distribution. + ### Stable version: -First, install the [required MMGen Wallet packages][7] for your Linux distribution: - -Then, - - $ python3 -m pip install --upgrade mmgen-node-tools +```bash +$ python3 -m pip install --upgrade mmgen-node-tools +``` ### Development version: -First, install the latest development version of [MMGen Wallet][6]. +Install the latest development version of [MMGen Wallet][6] for your platform. -Then, - - $ git clone https://github.com/mmgen/mmgen-node-tools - $ cd mmgen-node-tools - $ python3 -m build --no-isolation - $ python3 -m pip install dist/*.whl +```bash +$ git clone https://github.com/mmgen/mmgen-node-tools +$ cd mmgen-node-tools +$ python3 -m build --no-isolation +$ python3 -m pip install dist/*.whl +``` ## Test: *NOTE: the tests require that the MMGen Wallet and MMGen Node Tools repositories be located in the same directory.* +#### Windows/MSYS2: + +> *Tested only on NTFS – with ReFS your mileage may vary* + +> Turn on Developer Mode to enable symlinks: +``` + Settings -> Update & Security -> For developers -> Developer Mode: On +``` +> and add this to your `~/.bashrc`: +```bash + export MSYS=winsymlinks:nativestrict +``` +> Close and reopen the MSYS2 terminal to update your environment. + Initialize the test framework (must be run at least once after cloning, and possibly again after a pull if tests have been updated): @@ -49,11 +79,29 @@ Full testing: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -[**Forum**][4] | -[PGP Public Key][5] | -Donate: 15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w +Homepage: +[Clearnet](https://mmgen-wallet.cc) | +[I2P](http://mmgen-wallet.i2p) | +[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion) +Code repository: +[Clearnet](https://mmgen.org/project/mmgen/mmgen-wallet) | +[I2P](http://mmgen-wallet.i2p/project/mmgen/mmgen-wallet) | +[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion/project/mmgen/mmgen-wallet) +Code repository mirrors: +[Github](https://github.com/mmgen/mmgen-wallet) | +[Gitlab](https://gitlab.com/mmgen/mmgen-wallet) | +[Gitflic](https://gitflic.ru/project/mmgen/mmgen-wallet) +[Keybase](https://keybase.io/mmgen) | +[Reddit](https://www.reddit.com/user/mmgen-py) | +[Bitcointalk](https://bitcointalk.org/index.php?topic=567069.new#new) +[PGP Signing Key][5]: 5C84 CB45 AEE2 250F 31A6 A570 3F8B 1861 E32B 7DA2 +Donate: + ⊙ BTC: *bc1qxmymxf8p5ckvlxkmkwgw8ap5t2xuaffmrpexap* + ⊙ BCH: *15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w* + ⊙ XMR: *8B14zb8wgLuKDdse5p8f3aKpFqRdB4i4xj83b7BHYABHMvHifWxiDXeKRELnaxL5FySfeRRS5girgUvgy8fQKsYMEzPUJ8h* [4]: https://bitcointalk.org/index.php?topic=567069.0 [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ [7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux +[8]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Microsoft-Windows#a_m diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 21ca9e7..a91fcc8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev8 +3.2.dev9 diff --git a/setup.cfg b/setup.cfg index 4cb6a92..62f7074 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.dev22 + mmgen-wallet>=14.0.dev25 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index 5f5c691..7c38bcf 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -35,7 +35,7 @@ class CmdTestMain(CmdTestBase): "'mmnode-peerblocks' script", ('peerblocks1', '--help'), ('peerblocks2', 'interactive (popen spawn)'), - ('peerblocks3', 'interactive, 80 columns (pexpect_spawn)'), + ('peerblocks3', 'interactive, 80 columns (pexpect_spawn [on Linux])'), ), } @@ -94,4 +94,6 @@ class CmdTestMain(CmdTestBase): return t def peerblocks3(self): - return self.peerblocks2(['--columns=80'],pexpect_spawn=True) + return self.peerblocks2( + ['--columns=80'], + pexpect_spawn = sys.platform != 'win32' ) diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index b17b4c6..e554930 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -12,7 +12,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ -import os +import sys,os from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto @@ -134,6 +134,8 @@ class CmdTestRegtest(CmdTestBase): def netrate2(self): t = self.netrate( [], r'sent:.*' ) t.kill(15) + if sys.platform == 'win32': + return 'ok' t.req_exit_val = -15 return t From 4b22d67e37d22a73701ff7b90236389b448bda45 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 27 Nov 2023 09:32:24 +0000 Subject: [PATCH 046/108] update README.md --- README.md | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d3b18ba..4d3fe28 100644 --- a/README.md +++ b/README.md @@ -12,17 +12,16 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > Install [MSYS2 and the MMGen Wallet dependencies][8], skipping installation of > scrypt, libsecp256k1 and the wallet itself if desired. - +> > Install some additional dependencies: - -```bash - $ pacman -S \ - mingw-w64-ucrt-x86_64-python-pandas \ - mingw-w64-ucrt-x86_64-python-tqdm \ - mingw-w64-ucrt-x86_64-python-lxml - $ python3 -m pip install requests-futures - $ python3 -m pip install --no-deps yahooquery -``` +> ```bash +> $ pacman -S \ +> mingw-w64-ucrt-x86_64-python-pandas \ +> mingw-w64-ucrt-x86_64-python-tqdm \ +> mingw-w64-ucrt-x86_64-python-lxml +> $ python3 -m pip install requests-futures +> $ python3 -m pip install --no-deps yahooquery +> ``` #### Linux: @@ -53,15 +52,15 @@ located in the same directory.* #### Windows/MSYS2: > *Tested only on NTFS – with ReFS your mileage may vary* - +> > Turn on Developer Mode to enable symlinks: -``` - Settings -> Update & Security -> For developers -> Developer Mode: On -``` +> ``` +> Settings -> Update & Security -> For developers -> Developer Mode: On +> ``` > and add this to your `~/.bashrc`: -```bash - export MSYS=winsymlinks:nativestrict -``` +> ```bash +> export MSYS=winsymlinks:nativestrict +> ``` > Close and reopen the MSYS2 terminal to update your environment. Initialize the test framework (must be run at least once after cloning, and From ef0acc254225d712465cfca9558b30bbd628f41c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 27 Nov 2023 10:55:22 +0000 Subject: [PATCH 047/108] Version 3.2.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index a91fcc8..944880f 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev9 +3.2.0 diff --git a/setup.cfg b/setup.cfg index 62f7074..f737c62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.dev25 + mmgen-wallet==14.0.0 pyyaml yahooquery From 3dace6b188aaed1909f9bc4a6973df996efaecaa Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 30 Nov 2023 09:19:10 +0000 Subject: [PATCH 048/108] README.md: fix MSWin deps, fix repo URLs, add Twitter link --- README.md | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4d3fe28..f72051f 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,7 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > > Install some additional dependencies: > ```bash -> $ pacman -S \ -> mingw-w64-ucrt-x86_64-python-pandas \ -> mingw-w64-ucrt-x86_64-python-tqdm \ -> mingw-w64-ucrt-x86_64-python-lxml +> $ pacman -S mingw-w64-ucrt-x86_64-python-pandas > $ python3 -m pip install requests-futures > $ python3 -m pip install --no-deps yahooquery > ``` @@ -35,7 +32,7 @@ $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: -Install the latest development version of [MMGen Wallet][6] for your platform. +Install the latest development version of [MMGen Wallet][6] for your platform: ```bash $ git clone https://github.com/mmgen/mmgen-node-tools @@ -65,32 +62,34 @@ located in the same directory.* Initialize the test framework (must be run at least once after cloning, and possibly again after a pull if tests have been updated): - - $ test/init.sh - +``` +$ test/init.sh +``` BTC-only testing: - - $ test/test-release.sh -A - +``` +$ test/test-release.sh -A +``` Full testing: - - $ test/test-release.sh +``` +$ test/test-release.sh +``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Homepage: -[Clearnet](https://mmgen-wallet.cc) | +[Clearnet](https://mmgen.org) | [I2P](http://mmgen-wallet.i2p) | [Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion) Code repository: -[Clearnet](https://mmgen.org/project/mmgen/mmgen-wallet) | -[I2P](http://mmgen-wallet.i2p/project/mmgen/mmgen-wallet) | -[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion/project/mmgen/mmgen-wallet) +[Clearnet](https://mmgen.org/project/mmgen/mmgen-node-tools) | +[I2P](http://mmgen-wallet.i2p/project/mmgen/mmgen-node-tools) | +[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion/project/mmgen/mmgen-node-tools) Code repository mirrors: -[Github](https://github.com/mmgen/mmgen-wallet) | -[Gitlab](https://gitlab.com/mmgen/mmgen-wallet) | -[Gitflic](https://gitflic.ru/project/mmgen/mmgen-wallet) +[Github](https://github.com/mmgen/mmgen-node-tools) | +[Gitlab](https://gitlab.com/mmgen/mmgen-node-tools) | +[Gitflic](https://gitflic.ru/project/mmgen/mmgen-node-tools) [Keybase](https://keybase.io/mmgen) | +[Twitter](https://twitter.com/TheMMGenProject) | [Reddit](https://www.reddit.com/user/mmgen-py) | [Bitcointalk](https://bitcointalk.org/index.php?topic=567069.new#new) [PGP Signing Key][5]: 5C84 CB45 AEE2 250F 31A6 A570 3F8B 1861 E32B 7DA2 @@ -99,7 +98,6 @@ Donate:  ⊙ BCH: *15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w*  ⊙ XMR: *8B14zb8wgLuKDdse5p8f3aKpFqRdB4i4xj83b7BHYABHMvHifWxiDXeKRELnaxL5FySfeRRS5girgUvgy8fQKsYMEzPUJ8h* -[4]: https://bitcointalk.org/index.php?topic=567069.0 [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ [7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux From de29b213ff824ee247017c16c52a48378f1c5e17 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 30 Nov 2023 11:25:34 +0000 Subject: [PATCH 049/108] bump version and wallet dependency --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 944880f..0e5a99a 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.0 +3.3.dev0 diff --git a/setup.cfg b/setup.cfg index f737c62..7827398 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet==14.0.0 + mmgen-wallet>=14.0.0 pyyaml yahooquery From cc0f4729c6efa8db40e93b43d88f569cd9442c03 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 12 Dec 2023 09:58:01 +0000 Subject: [PATCH 050/108] update for mmgen-wallet v14.1.dev2 --- cmds/mmnode-addrbal | 2 +- cmds/mmnode-blocks-info | 2 +- cmds/mmnode-feeview | 2 +- cmds/mmnode-halving-calculator | 2 +- cmds/mmnode-netrate | 2 +- cmds/mmnode-peerblocks | 2 +- cmds/mmnode-ticker | 2 +- cmds/mmnode-txfind | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/unit_tests_d/ut_dep.py | 21 +++++++++++++++++++++ 11 files changed, 31 insertions(+), 10 deletions(-) create mode 100755 test/unit_tests_d/ut_dep.py diff --git a/cmds/mmnode-addrbal b/cmds/mmnode-addrbal index 3a5bc68..a14edc0 100755 --- a/cmds/mmnode-addrbal +++ b/cmds/mmnode-addrbal @@ -14,4 +14,4 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain from mmgen.main import launch -launch('addrbal',package='mmgen_node_tools') +launch(mod='addrbal',package='mmgen_node_tools') diff --git a/cmds/mmnode-blocks-info b/cmds/mmnode-blocks-info index a1c855e..d542d1a 100755 --- a/cmds/mmnode-blocks-info +++ b/cmds/mmnode-blocks-info @@ -14,4 +14,4 @@ mmnode-blocks-info: Display information about a block or range of blocks from mmgen.main import launch -launch('blocks_info',package='mmgen_node_tools') +launch(mod='blocks_info',package='mmgen_node_tools') diff --git a/cmds/mmnode-feeview b/cmds/mmnode-feeview index 5a4625e..8b639bb 100755 --- a/cmds/mmnode-feeview +++ b/cmds/mmnode-feeview @@ -14,4 +14,4 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool from mmgen.main import launch -launch('feeview',package='mmgen_node_tools') +launch(mod='feeview',package='mmgen_node_tools') diff --git a/cmds/mmnode-halving-calculator b/cmds/mmnode-halving-calculator index ebf1f6d..1759f77 100755 --- a/cmds/mmnode-halving-calculator +++ b/cmds/mmnode-halving-calculator @@ -14,4 +14,4 @@ mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) from mmgen.main import launch -launch('halving_calculator',package='mmgen_node_tools') +launch(mod='halving_calculator',package='mmgen_node_tools') diff --git a/cmds/mmnode-netrate b/cmds/mmnode-netrate index cfefab2..f8c4ad1 100755 --- a/cmds/mmnode-netrate +++ b/cmds/mmnode-netrate @@ -14,4 +14,4 @@ mmnode-netrate: Bitcoin daemon network rate monitor from mmgen.main import launch -launch('netrate',package='mmgen_node_tools') +launch(mod='netrate',package='mmgen_node_tools') diff --git a/cmds/mmnode-peerblocks b/cmds/mmnode-peerblocks index be4bb41..5a02741 100755 --- a/cmds/mmnode-peerblocks +++ b/cmds/mmnode-peerblocks @@ -14,4 +14,4 @@ mmnode-peerblocks: List blocks in flight, disconnect stalling nodes from mmgen.main import launch -launch('peerblocks',package='mmgen_node_tools') +launch(mod='peerblocks',package='mmgen_node_tools') diff --git a/cmds/mmnode-ticker b/cmds/mmnode-ticker index d6e91fb..f0160e7 100755 --- a/cmds/mmnode-ticker +++ b/cmds/mmnode-ticker @@ -14,4 +14,4 @@ mmnode-ticker: Display price information for cryptocurrency and other assets from mmgen.main import launch -launch('ticker',package='mmgen_node_tools') +launch(mod='ticker',package='mmgen_node_tools') diff --git a/cmds/mmnode-txfind b/cmds/mmnode-txfind index 05213a8..fd48cf4 100755 --- a/cmds/mmnode-txfind +++ b/cmds/mmnode-txfind @@ -14,4 +14,4 @@ mmnode-txfind: Find a transaction in the blockchain or mempool from mmgen.main import launch -launch('txfind',package='mmgen_node_tools') +launch(mod='txfind',package='mmgen_node_tools') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0e5a99a..f5f664c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev0 +3.3.dev1 diff --git a/setup.cfg b/setup.cfg index 7827398..c99a9ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.0 + mmgen-wallet>=14.1.dev2 pyyaml yahooquery diff --git a/test/unit_tests_d/ut_dep.py b/test/unit_tests_d/ut_dep.py new file mode 100755 index 0000000..641c3c3 --- /dev/null +++ b/test/unit_tests_d/ut_dep.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +""" +test.unit_tests_d.ut_dep: dependency unit tests for the MMGen Node Tools + + Test whether dependencies are installed and functional. + No data verification is performed. +""" + +from ..include.common import vmsg,imsg +from mmgen.color import yellow + +class unit_tests: + + def yahooquery(self,name,ut): + try: + from yahooquery import Ticker + return True + except ImportError: + imsg(yellow('Unable to import Ticker from yahooquery')) + return False From 6f2925f7973b9918a5582c35167bcf9ffe09b92c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 10 Feb 2024 15:15:18 +0000 Subject: [PATCH 051/108] test/init.sh, MANIFEST.in: fixes, cleanups --- MANIFEST.in | 10 ++++++++-- mmgen_node_tools/Ticker.py | 4 ++-- test/init.sh | 31 +++++++++++++++++++++---------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7bdab90..370da07 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,10 @@ include README.md LICENSE -include test/test-release.sh -include test/unit_tests_d/*.py + include mmgen_node_tools/data/* + +include test/init.sh +include test/test-release.d/*.sh +include test/unit_tests_d/*.py +include test/cmdtest_py_d/*.py +include test/overlay/fakemods/mmgen_node_tools/*.py +include test/ref/*/* diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 5a6a6a2..e8b704f 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -193,7 +193,7 @@ class DataSource: @property def timeout(self): - return 5 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit + return 0 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit def json_data_error_msg(self,json_text): tor_captcha_msg = f""" @@ -265,7 +265,7 @@ class DataSource: @property def timeout(self): - return 5 if gcfg.test_suite else self.ratelimit + return 0 if gcfg.test_suite else self.ratelimit @property def symbols(self): diff --git a/test/init.sh b/test/init.sh index 81727bb..51e4e9e 100755 --- a/test/init.sh +++ b/test/init.sh @@ -10,6 +10,7 @@ RED="\e[31;1m" GREEN="\e[32;1m" YELLOW="\e[33;1m" BLUE="\e[34;1m" RESET="\e[0m" +set -e set -o errtrace set -o functrace @@ -17,14 +18,19 @@ trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT trap 'echo -e "${RED}Node Tools test suite initialization exited with error (line $BASH_LINENO) $RESET"' ERR umask 0022 +STDOUT_DEVNULL='>/dev/null' +STDERR_DEVNULL='2>/dev/null' + PROGNAME=$(basename $0) -while getopts h OPT +while getopts hv OPT do case "$OPT" in h) printf " %-16s Initialize the MMGen Node Tools test suite\n" "${PROGNAME}:" echo " USAGE: $PROGNAME" echo " OPTIONS: '-h' Print this help message" + echo " -v Be more verbose" exit ;; + v) VERBOSE=1 STDOUT_DEVNULL='' STDERR_DEVNULL='' ;; *) exit ;; esac done @@ -41,7 +47,10 @@ check_mmgen_repo() { } build_mmgen_extmod() { - ( cd $wallet_repo; python3 ./setup.py build_ext --inplace ) + ( + cd $wallet_repo + eval "python3 ./setup.py build_ext --inplace $STDOUT_DEVNULL $STDERR_DEVNULL" + ) } create_dir_links() { @@ -49,14 +58,14 @@ create_dir_links() { target="$wallet_repo/$link_name" if [ -L $link_name ]; then [ "$(realpath --relative-to=. $link_name 2>/dev/null)" == $target ] || { - echo "Removing broken symlink '$link_name'" + [ "$VERBOSE" ] && echo "Removing broken symlink '$link_name'" rm $link_name } elif [ -e $link_name ]; then die "'$link_name' is not a symbolic link. Please remove or relocate it and re-run this script" fi if [ ! -e $link_name ]; then - echo "Creating symlink: $link_name" + [ "$VERBOSE" ] && echo "Creating symlink: $link_name" ln -s $target fi done @@ -87,32 +96,34 @@ create_test_links() { fs="%-8s %-16s %s -> %s\n" if [ $type == 'hard' ]; then if [ -L $path ]; then - printf "$fs" "Deleting" "symbolic link:" $path $target + [ "$VERBOSE" ] && printf "$fs" "Deleting" "symbolic link:" $path $target rm -rf $path elif [ -e $path ]; then if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then - printf "$fs" "Deleting" "stale hard link:" $path "?" + [ "$VERBOSE" ] && printf "$fs" "Deleting" "stale hard link:" $path "?" rm -rf $path fi fi fi if [ ! -e $path ]; then # link is either absent or a broken symlink - printf "$fs" "Creating" "$type link:" $path $target + [ "$VERBOSE" ] && printf "$fs" "Creating" "$type link:" $path $target ( cd "$(dirname $path)" && ln -f $symlink_arg $pfx$target ) fi done <<<$paths } -set -e - becho 'Initializing MMGen Node Tools Test Suite' check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod +[ "$VERBOSE" ] && becho 'Creating links to mmgen-wallet repo' + create_dir_links create_test_links -becho 'OK' +[ "$VERBOSE" ] && becho 'OK' + +true From 7c7c2c7da839ce3139c2260eb293d0e67fa1acf7 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 10 Feb 2024 15:15:18 +0000 Subject: [PATCH 052/108] mmnode-ticker: backport fix for Python 3.9 --- mmgen_node_tools/Ticker.py | 3 ++- mmgen_node_tools/data/version | 2 +- setup.cfg | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index e8b704f..33fb1fa 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -440,7 +440,8 @@ def gen_data(data): d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] - d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp()) + # .replace('Z','+00:00') -- Python 3.9 backport + d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated'].replace('Z','+00:00')).timestamp()) yield (d['id'],d) found[k].add(d[k]) wants[k].remove(d[k]) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f5f664c..0ed23d0 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev1 +3.3.dev2 diff --git a/setup.cfg b/setup.cfg index c99a9ef..82704a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,11 +24,11 @@ classifiers = Development Status :: 5 - Production/Stable [options] -python_requires = >=3.8 +python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev2 + mmgen-wallet>=14.1.dev8 pyyaml yahooquery From db4ba26dced535ac60465a71c7ff939305b3ae6a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 8 Mar 2024 15:32:54 +0000 Subject: [PATCH 053/108] update testing for MMGen Wallet v14.1.dev19 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 13 ++++++------- test/cmdtest_py_d/ct_regtest.py | 26 +++++++++++++++++--------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0ed23d0..3170c6e 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev2 +3.3.dev3 diff --git a/setup.cfg b/setup.cfg index 82704a4..f0c6629 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev8 + mmgen-wallet>=14.1.dev19 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 3dbac50..b4b83e9 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -106,10 +106,11 @@ class CmdTestScripts(CmdTestBase): shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' - def ticker(self,args=[],expect_list=None,cached=True): + def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): t = self.spawn( f'mmnode-ticker', - (['--cached-data'] if cached else []) + self.ticker_args + args ) + (['--cached-data'] if cached else []) + self.ticker_args + args, + exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) return t @@ -120,12 +121,11 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False) + t = self.ticker(cached=False, exit_val=3) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - ret = t.expect(['proxy host could not be resolved','ProxyError']) - t.req_exit_val = 3 if ret == 0 else 1 + t.expect(['proxy host could not be resolved', 'ProxyError']) return t def ticker3(self): @@ -165,9 +165,8 @@ class CmdTestScripts(CmdTestBase): return t def ticker6(self): - t = self.ticker( ['--wide','--portfolio'], None ) + t = self.ticker(['--wide','--portfolio'], None, exit_val=1) t.expect('No portfolio') - t.req_exit_val = 1 return t def ticker7(self): # demo diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index e554930..c89f8fa 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -39,6 +39,8 @@ class CmdTestRegtest(CmdTestBase): tmpdir_nums = [1] color = True deterministic = False + bdb_wallet = True + cmd_group_in = ( ('setup', 'regtest mode setup'), ('subgroup.netrate', []), @@ -111,20 +113,27 @@ class CmdTestRegtest(CmdTestBase): die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) - self.regtest = MMGenRegtest(cfg,self.proto.coin) + + self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' + self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) from shutil import rmtree - try: rmtree(joinpath(self.tr.data_dir,'regtest')) - except: pass - t = self.spawn('mmgen-regtest',['-n','setup']) + try: + rmtree(joinpath(self.tr.data_dir,'regtest')) + except: + pass + t = self.spawn( + 'mmgen-regtest', + (['--bdb-wallet'] if self.use_bdb_wallet else []) + + ['--setup-no-stop-daemon', 'setup']) for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): t.expect(s) return t - def netrate(self,add_args,expect_str): - t = self.spawn( 'mmnode-netrate', args1 + add_args ) + def netrate(self, add_args, expect_str, exit_val=None): + t = self.spawn('mmnode-netrate', args1 + add_args, exit_val=exit_val) t.expect(expect_str,regex=True) return t @@ -132,11 +141,10 @@ class CmdTestRegtest(CmdTestBase): return self.netrate( ['--help'], 'USAGE:.*' ) def netrate2(self): - t = self.netrate( [], r'sent:.*' ) + t = self.netrate([], r'sent:.*', exit_val=-15) t.kill(15) if sys.platform == 'win32': return 'ok' - t.req_exit_val = -15 return t def halving_calculator(self,add_args,expect_list): @@ -326,7 +334,7 @@ class CmdTestRegtest(CmdTestBase): return await do_tx( [{ 'txid': tx_input['txid'], 'vout': 0 }], outputs, - r.miner_wif ) + await r.miner_wif) async def do_tx2(tx,pairno): fee = fees[pairno] From c4ace71049ca48e00bdd0a6d204df96ddf71526c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 10 Mar 2024 14:44:57 +0000 Subject: [PATCH 054/108] minor fixes and cleanups --- mmgen_node_tools/Ticker.py | 16 ++++++++++++---- test/cmdtest_py_d/ct_misc.py | 5 +++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 33fb1fa..806b0e9 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -26,6 +26,7 @@ from collections import namedtuple from mmgen.color import red,yellow,green,blue,orange,gray from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.ui import do_pager homedir = os.getenv('HOME') dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') @@ -407,6 +408,8 @@ def gen_data(data): id = get_id(k,v) if wants['id']: if id in wants['id']: + if not isinstance(v,dict): + die(2, str(v)) if id in found['id']: die(1,dup_sym_errmsg(id)) if m := data['hi'].get(k): @@ -517,7 +520,6 @@ def main(): return if gcfg.list_ids: - from mmgen.ui import do_pager do_pager('\n'.join(e['id'] for e in src_data['cc'])) return @@ -628,9 +630,10 @@ def make_cfg(gcfg_arg): def get_portfolio(): return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'} - def parse_add_precision(s): - if not s: + def parse_add_precision(arg): + if not arg: return 0 + s = str(arg) if not (s.isdigit() and s.isascii()): die(1,f'{s}: invalid parameter for --add-precision (not an integer)') if int(s) > 30: @@ -706,7 +709,12 @@ def make_cfg(gcfg_arg): cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None, + portfolio = + get_portfolio() + if cfg_in.portfolio + and gcfg.portfolio + and not query + else None, percent_cols = parse_percent_cols(gcfg.percent_cols) ) diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index b4b83e9..8127e1d 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -121,11 +121,12 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False, exit_val=3) + t = self.ticker(cached=False) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - t.expect(['proxy host could not be resolved', 'ProxyError']) + ret = t.expect(['proxy host could not be resolved', 'ProxyError']) + t.exit_val = 1 if ret else 3 return t def ticker3(self): From 867026e8edd370dbbf655d2ea20ad384a50d4f4e Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 10 Mar 2024 14:44:57 +0000 Subject: [PATCH 055/108] mmnode-ticker: add config file vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following command-line opts are now configurable in ‘ticker-cfg.yaml’: add_precision asset_limit btc cached_data elapsed name_labels pager percent_cols thousands_comma update_time verbose --- mmgen_node_tools/Ticker.py | 65 ++++++++++++++++++++++------------- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 806b0e9..fd05f99 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red,yellow,green,blue,orange,gray -from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.util import msg,msg_r,Msg,Msg_r,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -75,7 +75,7 @@ class DataSource: ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], ['--compressed'], # adds 'Accept-Encoding: gzip' ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], - ['--silent', not gcfg.verbose], + ['--silent', not cfg.verbose], [self.api_url] ) @@ -101,7 +101,9 @@ class DataSource: if not os.path.exists(self.json_fn): open(self.json_fn,'w').write('{}') - if gcfg.cached_data: + use_cached_data = cfg.cached_data and not gcfg.download + + if use_cached_data: data_type = 'json' data_in = open(self.json_fn).read() else: @@ -112,8 +114,8 @@ class DataSource: msg('') self.fetch_delay() msg_r(f'Fetching {self.data_desc} from {self.api_host}...') - if self.has_verbose: - gcfg._util.vmsg('') + if self.has_verbose and cfg.verbose: + msg('') data_in = self.get_data_from_network() msg('done') if gcfg.testing: @@ -133,14 +135,14 @@ class DataSource: json_text = json.dumps(data_in) if not data: - if gcfg.cached_data: + if use_cached_data: die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') else: die(2,'Remote host returned no data!') elif 'error' in data: die(1,data['error']) - if gcfg.cached_data: + if use_cached_data: msg(f'Using cached data from ~/{self.json_fn_rel}') else: open(self.json_fn,'w').write(json_text) @@ -171,7 +173,7 @@ class DataSource: dfl_asset_limit = 2000 def __init__(self): - self.asset_limit = int(gcfg.asset_limit or self.dfl_asset_limit) + self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self,elapsed): return ( @@ -528,9 +530,8 @@ def main(): data = dict(gen_data(src_data)) - gcfg._util.stdout_or_pager( - '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n' - ) + (do_pager if cfg.pager else Msg_r)( + '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n') def make_cfg(gcfg_arg): @@ -670,7 +671,15 @@ def make_cfg(gcfg_arg): 'proxy', 'proxy2', 'portfolio', - 'percent_cols' ]) + 'percent_cols', + 'asset_limit', + 'cached_data', + 'elapsed', + 'name_labels', + 'pager', + 'thousands_comma', + 'update_time', + 'verbose']) global gcfg,cfg_in,src_cls,cfg @@ -704,18 +713,26 @@ def make_cfg(gcfg_arg): query = query, adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ), clsname = 'trading' if query else 'overview', - btc_only = gcfg.btc, - add_prec = parse_add_precision(gcfg.add_precision), + btc_only = gcfg.btc or cfg_in.cfg.get('btc'), + add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = get_portfolio() if cfg_in.portfolio - and gcfg.portfolio + and (gcfg.portfolio or cfg_in.cfg.get('portfolio')) and not query else None, - percent_cols = parse_percent_cols(gcfg.percent_cols) + percent_cols = parse_percent_cols(gcfg.percent_cols or cfg_in.cfg.get('percent_cols')), + asset_limit = gcfg.asset_limit or cfg_in.cfg.get('asset_limit'), + cached_data = gcfg.cached_data or cfg_in.cfg.get('cached_data'), + elapsed = gcfg.elapsed or cfg_in.cfg.get('elapsed'), + name_labels = gcfg.name_labels or cfg_in.cfg.get('name_labels'), + pager = gcfg.pager or cfg_in.cfg.get('pager'), + thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), + update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), + verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), ) def get_cfg_in(): @@ -753,10 +770,10 @@ class Ticker: def __init__(self,data): - self.comma = ',' if gcfg.thousands_comma else '' + self.comma = ',' if cfg.thousands_comma else '' self.col1_wid = max(len('TOTAL'),( - max(len(self.create_label(d['id'])) for d in data.values()) if gcfg.name_labels else + max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else max(len(d['symbol']) for d in data.values()) )) + 1 @@ -769,7 +786,7 @@ class Ticker: def format_last_update_col(self,cross_assets=()): - if gcfg.elapsed: + if cfg.elapsed: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: @@ -918,7 +935,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( - lbl = (self.create_label(d['id']) if gcfg.name_labels else d['symbol']), + lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), pc3 = fmt_pct(d.get('percent_change_1y')), @@ -969,11 +986,11 @@ class Ticker: ( 'pct1m', 'm' in cfg.percent_cols ), ( 'pct1w', 'w' in cfg.percent_cols ), ( 'pct1d', 'd' in cfg.percent_cols ), - ( 'update_time', gcfg.update_time ), + ( 'update_time', cfg.update_time ), ) if b] ) cols2 = list(cols) - if gcfg.update_time: + if cfg.update_time: cols2.pop() cols2.append('amt') @@ -1058,7 +1075,7 @@ class Ticker: if self.show_adj: self.fs_str += ' {p_adj}' self.hl_wid += self.max_wid + 1 - if gcfg.update_time: + if cfg.update_time: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 @@ -1071,7 +1088,7 @@ class Ticker: if self.show_adj else '' ) return self.fs_str.format( - lbl = (self.create_label(id) if gcfg.name_labels else d['symbol']), + lbl = self.create_label(id) if cfg.name_labels else d['symbol'], p_spot = green(p_spot) if id in self.hl_ids else p_spot, p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, upd = d.get('last_updated_fmt'), diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 3170c6e..1876a63 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev3 +3.3.dev4 diff --git a/setup.cfg b/setup.cfg index f0c6629..e3ef90e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev19 + mmgen-wallet>=14.1.dev21 pyyaml yahooquery From 18a92cd461c88cc1b1b48b3cf8f969d20fc79088 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 19 Jul 2024 09:53:52 +0000 Subject: [PATCH 056/108] mmnode-ticker: add `--quiet` option, back up cached JSON data --- mmgen_node_tools/Ticker.py | 17 +++++++++++++---- mmgen_node_tools/main_ticker.py | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index fd05f99..bae27c2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -136,17 +136,24 @@ class DataSource: if not data: if use_cached_data: - die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') + die(1, + f'No cached {self.data_desc}! Run command without the --cached-data option, ' + 'or use --download to retrieve data from remote host') else: die(2,'Remote host returned no data!') elif 'error' in data: die(1,data['error']) if use_cached_data: - msg(f'Using cached data from ~/{self.json_fn_rel}') + if not cfg.quiet: + msg(f'Using cached data from ~/{self.json_fn_rel}') else: - open(self.json_fn,'w').write(json_text) - msg(f'JSON data cached to ~/{self.json_fn_rel}') + if os.path.exists(self.json_fn): + os.rename(self.json_fn, self.json_fn + '.bak') + with open(self.json_fn, 'w') as fh: + fh.write(json_text) + if not cfg.quiet: + msg(f'JSON data cached to ~/{self.json_fn_rel}') if gcfg.download: sys.exit(0) @@ -679,6 +686,7 @@ def make_cfg(gcfg_arg): 'pager', 'thousands_comma', 'update_time', + 'quiet', 'verbose']) global gcfg,cfg_in,src_cls,cfg @@ -732,6 +740,7 @@ def make_cfg(gcfg_arg): pager = gcfg.pager or cfg_in.cfg.get('pager'), thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), + quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), ) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index f758089..c3f7325 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -53,6 +53,7 @@ opts_data = { columns ‘C’ (specify with comma-separated letters {pc}) -P, --pager Pipe the output to a pager +-q, --quiet Produce quieter output -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. From 0c54cb2c974943ef3a2301b3c6f5892af1e32cf8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 19 Jul 2024 09:54:03 +0000 Subject: [PATCH 057/108] Version 3.3.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1876a63..15a2799 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev4 +3.3.0 diff --git a/setup.cfg b/setup.cfg index e3ef90e..1945c49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev21 + mmgen-wallet==14.1.0 pyyaml yahooquery From 1a9184debc2f09205abd13a406619e368d7c54dc Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 20 Jul 2024 16:45:56 +0000 Subject: [PATCH 058/108] version bump --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 15a2799..49cbc5f 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.0 +3.4.dev0 diff --git a/setup.cfg b/setup.cfg index 1945c49..def5f65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet==14.1.0 + mmgen-wallet>=14.1.0 pyyaml yahooquery From c9d6d8f0475f003a7a86f7d5d005ed94495d5bbd Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 22 Sep 2024 16:10:08 +0000 Subject: [PATCH 059/108] Version 3.4.0 - released concurrently with MMGen Wallet v15.0.0 - adds macOS support --- README.md | 6 +++--- mmgen_node_tools/data/keywords | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 16 +++++++++++++--- test/cmdtest_py_d/cfg.py | 7 ++++++- test/cmdtest_py_d/ct_main.py | 2 +- test/init.sh | 13 ++++++++++--- 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f72051f..2751da7 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > $ python3 -m pip install --no-deps yahooquery > ``` -#### Linux: +#### Linux, macOS: -> Install the [required MMGen Wallet packages][7] for your Linux distribution. +> Install some [required packages][7] with your package manager and pip. ### Stable version: @@ -100,5 +100,5 @@ Donate: [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ -[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux +[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-Wallet-on-Linux-or-macOS [8]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Microsoft-Windows#a_m diff --git a/mmgen_node_tools/data/keywords b/mmgen_node_tools/data/keywords index 3c3c563..f26b502 100644 --- a/mmgen_node_tools/data/keywords +++ b/mmgen_node_tools/data/keywords @@ -1 +1 @@ -Bitcoin, BTC, Ethereum, ETH, Monero, XMR, ERC20, cryptocurrency, wallet, BIP32, cold storage, offline, online, spending, open-source, command-line, Python, Linux, Bitcoin Core, bitcoind, hd, deterministic, hierarchical, secure, anonymous, Electrum, seed, mnemonic, brainwallet, Scrypt, utility, script, scriptable, blockchain, raw, transaction, permissionless, console, terminal, curses, ansi, color, tmux, remote, client, daemon, RPC, json, entropy, xterm, rxvt, PowerShell, MSYS, MSYS2, MinGW, MinGW64, MSWin, Armbian, Raspbian, Raspberry Pi, Orange Pi, BCash, BCH, Litecoin, LTC, altcoin, ZEC, Zcash, DASH, Dashpay, SHA256Compress, monerod, EMC, Emercoin, token, deploy, contract, gas, fee, smart contract, solidity, Parity, OpenEthereum, testnet, devmode, Kovan +Bitcoin, BTC, Ethereum, ETH, Monero, XMR, ERC20, cryptocurrency, wallet, cold storage, offline, signing, online, security, privacy, spending, financial, investment, open-source, command-line, Python, Linux, Microsoft Windows, macOS, Bitcoin Core, BIP32, BIP39, BIP44, BIP69, BIP125, bitcoind, hd, deterministic, hierarchical, secure, anonymous, Electrum, seed, mnemonic, brainwallet, Scrypt, utility, script, scriptable, blockchain, raw, transaction, permissionless, console, terminal, curses, ansi, color, tmux, remote, client, daemon, RPC, json, entropy, xterm, rxvt, MSYS2, MSWin, Armbian, Raspbian, Raspberry Pi, Orange Pi, Rock Pi, BCash, Bitcoin Cash Node, BCH, Litecoin, LTC, altcoin, ZEC, Zcash, SHA256Compress, monerod, token, deploy, contract, gas, fee, smart contract, solidity, Parity, OpenEthereum, testnet, devmode, regtest diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 49cbc5f..1809198 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.4.dev0 +3.4.0 diff --git a/setup.cfg b/setup.cfg index def5f65..9a0ab04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,19 +8,29 @@ author = The MMGen Project author_email = mmgen@tuta.io url = https://github.com/mmgen/mmgen-node-tools license = GNU GPL v3 -platforms = Linux, Armbian, Raspbian, MS Windows +platforms = Linux, Armbian, Raspbian, MS Windows, MacOS keywords = file: mmgen_node_tools/data/keywords project_urls = Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = - Programming Language :: Python :: 3 License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows + Operating System :: MacOS Environment :: Console + Programming Language :: Python :: 3 + Programming Language :: C + Framework :: AsyncIO + Framework :: aiohttp Topic :: Office/Business :: Financial Topic :: Security :: Cryptography + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + Intended Audience :: Developers + Intended Audience :: End Users/Desktop + Intended Audience :: Financial and Insurance Industry + Intended Audience :: System Administrators Development Status :: 5 - Production/Stable [options] @@ -28,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.0 + mmgen-wallet==15.0.0 pyyaml yahooquery diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_py_d/cfg.py index 96bfe9f..ef52880 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -30,4 +30,9 @@ cfgs = { } def fixup_cfgs(): - pass + import os + + for k in cfgs: + cfgs[k]['tmpdir'] = os.path.join('test', 'tmp', str(k)) + +fixup_cfgs() diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index 7c38bcf..d52a2e3 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -96,4 +96,4 @@ class CmdTestMain(CmdTestBase): def peerblocks3(self): return self.peerblocks2( ['--columns=80'], - pexpect_spawn = sys.platform != 'win32' ) + pexpect_spawn = sys.platform == 'linux') diff --git a/test/init.sh b/test/init.sh index 51e4e9e..a379adf 100755 --- a/test/init.sh +++ b/test/init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet # Copyright (C)2013-2022 The MMGen Project @@ -18,6 +18,12 @@ trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT trap 'echo -e "${RED}Node Tools test suite initialization exited with error (line $BASH_LINENO) $RESET"' ERR umask 0022 +for i in '-c' '-f'; do + stat $i %i / >/dev/null 2>&1 && stat_fmt_opt=$i +done + +[ "$stat_fmt_opt" ] || { echo 'No suitable ‘stat’ binary found. Cannot proceed'; exit; } + STDOUT_DEVNULL='>/dev/null' STDERR_DEVNULL='2>/dev/null' @@ -77,6 +83,7 @@ create_test_links() { test/overlay/__init__.py symbolic test/overlay/fakemods/mmgen symbolic test/__init__.py symbolic + test/clean.py symbolic test/cmdtest.py hard test/unit_tests.py hard test/test-release.sh symbolic @@ -87,7 +94,7 @@ create_test_links() { while read path type; do [ "$path" ] || continue pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') - symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) + symlink_arg=$(if [ $type == 'symbolic' ]; then echo -s; fi) target="$wallet_repo/$path" if [ ! -e "$target" ]; then echo "Target path $target is missing! Cannot proceed" @@ -99,7 +106,7 @@ create_test_links() { [ "$VERBOSE" ] && printf "$fs" "Deleting" "symbolic link:" $path $target rm -rf $path elif [ -e $path ]; then - if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then + if [ "$(stat $stat_fmt_opt %i $path)" -ne "$(stat $stat_fmt_opt %i $target)" ]; then [ "$VERBOSE" ] && printf "$fs" "Deleting" "stale hard link:" $path "?" rm -rf $path fi From 92fdfc047e213aed42fdb7a71c29aef32707404f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 29 Sep 2024 14:29:10 +0000 Subject: [PATCH 060/108] Support BCH cashaddr format --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- setup.cfg | 2 +- test/cmdtest_py_d/ct_regtest.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1809198..3f8a888 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.4.0 +3.5.dev1 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7af2b77..e78a53e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -38,7 +38,7 @@ def do_output(proto,addr_data,blk_hdrs): indent = ' ' * (col1w + 2) for n,(addr,unspents) in enumerate(addr_data.items(),1): - Msg(f'\n{n:{col1w}}) Address: {addr.hl()}') + Msg(f'\n{n:{col1w}}) Address: {addr.hl(addr.view_pref)}') if unspents: heights = { u['height'] for u in unspents } @@ -93,7 +93,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): if unspents: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(width=max_addrw,color=True), + a = addr.fmt(addr.view_pref, width=max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), @@ -104,7 +104,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): else: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(width=max_addrw,color=True), + a = addr.fmt(addr.view_pref, width=max_addrw, color=True), u = ' -', b = '-', t = '', diff --git a/setup.cfg b/setup.cfg index 9a0ab04..875bb01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet==15.0.0 + mmgen-wallet>=15.1.dev1 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index c89f8fa..ee59b31 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -112,7 +112,7 @@ class CmdTestRegtest(CmdTestBase): if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) - self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) + self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto,'regtest',[1,2,3,4,5])] self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) From cb42eaa8cdc602be0d3718d73331b94745558d36 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 8 Oct 2024 13:19:15 +0000 Subject: [PATCH 061/108] update for MMGen Wallet 15.1.dev3 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 2 +- test/cmdtest_py_d/ct_regtest.py | 93 +++++++++++++++++---------------- 5 files changed, 53 insertions(+), 48 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 3f8a888..ecc3a64 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev1 +3.5.dev2 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c3f7325..351cb1e 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -234,7 +234,7 @@ from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config from . import Ticker -gcfg = Config( opts_data=opts_data, do_post_init=True ) +gcfg = Config(opts_data=opts_data, caller_post_init=True) Ticker.make_cfg(gcfg) diff --git a/setup.cfg b/setup.cfg index 875bb01..9b82f69 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev1 + mmgen-wallet>=15.1.dev3 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 8127e1d..6ac7d37 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -50,7 +50,7 @@ class CmdTestHelp(CmdTestBase): return t def longhelpscreens(self): - return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') + return self.helpscreens(arg='--longhelp',expect='USAGE:.*GLOBAL OPTIONS:') class CmdTestScripts(CmdTestBase): 'scripts not requiring a coin daemon' diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index ee59b31..0807a16 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -35,7 +35,6 @@ class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' networks = ('btc','ltc','bch') passthru_opts = ('coin',) - extra_spawn_args = ['--regtest=1'] tmpdir_nums = [1] color = True deterministic = False @@ -177,57 +176,57 @@ class CmdTestRegtest(CmdTestBase): def sendto2(self): return self.sendto(self.addrs[0],'0.234') def sendto3(self): return self.sendto(self.addrs[1],'0.345') - def addrbal(self,args,expect_list): - t = self.spawn('mmnode-addrbal',args) + def addrbal(self, args, expect_list): + t = self.spawn('mmnode-addrbal', args2 + args) t.match_expect_list(expect_list) return t def addrbal_single(self): return self.addrbal( - args2 + [self.addrs[0]], + [self.addrs[0]], [ f'Balance: 0.357 {cfg.coin}', '2 unspent outputs in 2 blocks', - '394','0.123', - '395','0.234' + '394', '0.123', + '395', '0.234' ]) def addrbal_multiple(self): return self.addrbal( - args2 + [self.addrs[1],self.addrs[0]], + [self.addrs[1], self.addrs[0]], [ - '396','0.345', - '394','0.123', - '395','0.234' + '396', '0.345', + '394', '0.123', + '395', '0.234' ]) def addrbal_multiple_tabular1(self): return self.addrbal( - args2 + ['--tabular',self.addrs[1],self.addrs[0]], + ['--tabular', self.addrs[1], self.addrs[0]], [ - self.addrs[1] + ' 1 396','0.345', - self.addrs[0] + ' 2 395','0.357' + self.addrs[1] + ' 1 396', '0.345', + self.addrs[0] + ' 2 395', '0.357' ]) def addrbal_multiple_tabular2(self): return self.addrbal( - args2 + ['--tabular','--first-block',self.addrs[1],self.addrs[0]], + ['--tabular', '--first-block', self.addrs[1], self.addrs[0]], [ - self.addrs[1] + ' 1 396','396','0.345', - self.addrs[0] + ' 2 394','395','0.357' + self.addrs[1] + ' 1 396', '396', '0.345', + self.addrs[0] + ' 2 394', '395', '0.357' ]) def addrbal_nobal1(self): return self.addrbal( - args2 + [self.addrs[2]], ['Address has no balance'] ) + [self.addrs[2]], ['Address has no balance']) def addrbal_nobal2(self): return self.addrbal( - args2 + [self.addrs[2],self.addrs[3]], ['Addresses have no balances'] ) + [self.addrs[2], self.addrs[3]], ['Addresses have no balances']) def addrbal_nobal3(self): return self.addrbal( - args2 + [self.addrs[4],self.addrs[0],self.addrs[3]], + [self.addrs[4], self.addrs[0], self.addrs[3]], [ 'No balance', '2 unspent outputs in 2 blocks', @@ -237,7 +236,7 @@ class CmdTestRegtest(CmdTestBase): def addrbal_nobal3_tabular1(self): return self.addrbal( - args2 + ['--tabular',self.addrs[4],self.addrs[0],self.addrs[3]], + ['--tabular', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - -', self.addrs[0] + ' 2 395','0.357', @@ -246,7 +245,7 @@ class CmdTestRegtest(CmdTestBase): def addrbal_nobal3_tabular2(self): return self.addrbal( - args2 + ['--tabular','--first-block',self.addrs[4],self.addrs[0],self.addrs[3]], + ['--tabular', '--first-block', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - - -', self.addrs[0] + ' 2 394','395','0.357', @@ -254,39 +253,45 @@ class CmdTestRegtest(CmdTestBase): ]) def blocks_info(self,args,expect_list): - t = self.spawn('mmnode-blocks-info',args) + t = self.spawn('mmnode-blocks-info', args1 + args) t.match_expect_list(expect_list) return t def blocks_info1(self): - return self.blocks_info( args1 + ['--help'], ['USAGE:','OPTIONS:']) + return self.blocks_info( + ['--help'], + ['USAGE:','OPTIONS:']) def blocks_info2(self): - return self.blocks_info( args1, [ - 'Current height: 396', - ]) + return self.blocks_info( + [], + ['Current height: 396']) def blocks_info3(self): - return self.blocks_info( args1 + ['+100'], [ - 'Range: 297-396', - 'Current height: 396', - 'Next diff adjust: 2016' - ]) + return self.blocks_info( + ['+100'], + [ + 'Range: 297-396', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) def blocks_info4(self): n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) - return self.blocks_info( args1 + ['--miner-info','--fields=all','--stats=all','+3'], [ - 'Averages', - f'nTx: {n1}', - f'Inputs: {i1}', - f'Outputs: {o1}', - 'Totals', - f'nTx: {n2}', - f'Inputs: {i2}', - f'Outputs: {o2}', - 'Current height: 396', - 'Next diff adjust: 2016' - ]) + return self.blocks_info( + ['--miner-info', '--fields=all', '--stats=all', '+3'], + [ + 'Averages', + f'nTx: {n1}', + f'Inputs: {i1}', + f'Outputs: {o1}', + 'Totals', + f'nTx: {n2}', + f'Inputs: {i2}', + f'Outputs: {o2}', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) async def feeview_setup(self): @@ -380,7 +385,7 @@ class CmdTestRegtest(CmdTestBase): return 'ok' def _feeview(self,args,expect_list=[]): - t = self.spawn('mmnode-feeview',args) + t = self.spawn('mmnode-feeview', args1 + args) if expect_list: t.match_expect_list(expect_list) return t From dee57d88869a8c9c6c22bbf01bc07de35de7545e Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:47 +0000 Subject: [PATCH 062/108] update for MMGen Wallet CoinAmt changes --- mmgen_node_tools/BlocksInfo.py | 27 +++++++++++++++++++-------- mmgen_node_tools/main_addrbal.py | 4 ++-- test/cmdtest_py_d/ct_regtest.py | 9 +++++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 40cc6e5..26e6cfa 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -23,6 +23,7 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of block import re,json from collections import namedtuple from time import strftime,gmtime +from decimal import Decimal from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int from mmgen.rpc import json_encoder @@ -241,14 +242,14 @@ class BlocksInfo: 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), 'fe': lambda arg: str(arg), - 'di': lambda arg: '{:.2e}'.format(arg), + 'di': lambda arg: '{:.2e}'.format(Decimal(arg)), } if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), - 'fe': lambda arg: str(int(arg * to_satoshi)), - 'tf': lambda arg: '{:.8f}'.format(arg), + 'fe': lambda arg: str(int(Decimal(arg) * to_satoshi)), + 'tf': lambda arg: '{:.8f}'.format(Decimal(arg)), }) self.fnames = tuple( @@ -547,7 +548,7 @@ class BlocksInfo: else: sample_blks = min(min_sample_blks,self.tip) start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) - diff_adj = float(tip_hdr['difficulty'] / start_hdr['difficulty']) + diff_adj = Decimal(tip_hdr['difficulty']) / Decimal(start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] bdi = ((time1 * diff_adj) + time2) / sample_blks @@ -570,17 +571,26 @@ class BlocksInfo: 'sample_blks': ('{}', sample_blks) } ), - ('Cur difficulty: {}', 'cur_diff', '{:.2e}', tip_hdr['difficulty']), + ('Cur difficulty: {}', 'cur_diff', '{:.2e}', Decimal(tip_hdr['difficulty'])), ('Est. diff adjust: {}%', 'est_diff_adjust_pct', '{:+.2f}', ((600 / bdi) - 1) * 100), )) + def sum_field_avg(self, field): + return self.sum_field_total(field) // len(self.res) + + def sum_field_total(self, field): + if isinstance(getattr(self.res[0], field), str): + return sum(Decimal(getattr(block, field)) for block in self.res) + else: + return sum(getattr(block, field) for block in self.res) + async def create_col_avg_stats(self): def gen(): for field in self.fnames: if field in self.avg_stats_skip: yield ( field, ('{}','') ) else: - ret = sum(getattr(block,field) for block in self.res) // len(self.res) + ret = self.sum_field_avg(field) func = self.fields[field].fmt_func yield ( field, ( (self.fmt_funcs[func] if func else '{}'), ret )) if not self.header_printed: @@ -590,9 +600,10 @@ class BlocksInfo: def avg_stats_data(self,data,spec_conv,spec_val): coin = self.rpc.proto.coin + return data( hdr = 'Averages for processed blocks:', - func = lambda field: sum(getattr(block,field) for block in self.res) // len(self.res), + func = self.sum_field_avg, spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}' }, spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), @@ -616,7 +627,7 @@ class BlocksInfo: coin = self.rpc.proto.coin return data( hdr = 'Totals for processed blocks:', - func = lambda field: sum(getattr(block,field) for block in self.res), + func = self.sum_field_total, spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}' }, spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index e78a53e..7dd37a3 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -44,7 +44,7 @@ def do_output(proto,addr_data,blk_hdrs): heights = { u['height'] for u in unspents } Msg('{}Balance: {}'.format( indent, - proto.coin_amt(sum(u['amount'] for u in unspents)).hl2(unit=True,fs='{:,}') )), + sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), Msg('{}{} unspent output{} in {} block{}'.format( indent, red(str(len(unspents))), @@ -99,7 +99,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = proto.coin_amt(sum(u['amount'] for u in unspents)).fmt(color=True,iwidth=7,prec=8) + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(color=True, iwidth=7, prec=8) )) else: Msg(fs.format( diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index 0807a16..f1b2774 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -13,6 +13,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import sys,os +from decimal import Decimal from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto @@ -316,13 +317,13 @@ class CmdTestRegtest(CmdTestBase): # very approximate tx size estimation: ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) - x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt(self.proto.coin_amt.satoshi) + x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt.satoshi n = n_in - 1 vmax = high - low for i in range(n_in): - yield (low + (i/n)**6 * vmax) * x + yield Decimal(low + (i/n)**6 * vmax) * x async def do_tx(inputs,outputs,wif): tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) @@ -335,14 +336,14 @@ class CmdTestRegtest(CmdTestBase): tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} - outputs.update({burn_addr: tx_input['amount'] - (tx1_amt*nTxs) - fee}) + outputs.update({burn_addr: self.proto.coin_amt(tx_input['amount']) - (tx1_amt*nTxs) - fee}) return await do_tx( [{ 'txid': tx_input['txid'], 'vout': 0 }], outputs, await r.miner_wif) async def do_tx2(tx,pairno): - fee = fees[pairno] + fee = self.proto.coin_amt(fees[pairno], from_decimal=True) outputs = {p.addr:tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) return await do_tx( From b9b289c56db227cfc42e541dffc7767bce129c78 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:48 +0000 Subject: [PATCH 063/108] directory rename: - test/cmdtest_py_d -> test/cmdtest_d --- MANIFEST.in | 2 +- test/{cmdtest_py_d => cmdtest_d}/cfg.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_main.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_misc.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_regtest.py | 2 +- test/init.sh | 4 ++-- test/test-release.d/cfg.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename test/{cmdtest_py_d => cmdtest_d}/cfg.py (93%) rename test/{cmdtest_py_d => cmdtest_d}/ct_main.py (96%) rename test/{cmdtest_py_d => cmdtest_d}/ct_misc.py (99%) rename test/{cmdtest_py_d => cmdtest_d}/ct_regtest.py (99%) diff --git a/MANIFEST.in b/MANIFEST.in index 370da07..1fe78fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,6 @@ include mmgen_node_tools/data/* include test/init.sh include test/test-release.d/*.sh include test/unit_tests_d/*.py -include test/cmdtest_py_d/*.py +include test/cmdtest_d/*.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_d/cfg.py similarity index 93% rename from test/cmdtest_py_d/cfg.py rename to test/cmdtest_d/cfg.py index ef52880..c28ce5d 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_d/cfg.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.cfg: configuration data for cmdtest.py +test.cmdtest_d.cfg: configuration data for cmdtest.py """ cmd_groups_altcoin = [] diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_d/ct_main.py similarity index 96% rename from test/cmdtest_py_d/ct_main.py rename to test/cmdtest_d/ct_main.py index d52a2e3..3009d2f 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_d/ct_main.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-wallet """ -cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite +cmdtest_d.ct_main: Basic operations tests for the cmdtest.py test suite """ import sys,time diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_d/ct_misc.py similarity index 99% rename from test/cmdtest_py_d/ct_misc.py rename to test/cmdtest_d/ct_misc.py index 6ac7d37..15b109e 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_d/ct_misc.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite +test.cmdtest_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite """ import os,shutil diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_d/ct_regtest.py similarity index 99% rename from test/cmdtest_py_d/ct_regtest.py rename to test/cmdtest_d/ct_regtest.py index f1b2774..24f19b9 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_d/ct_regtest.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite +test.cmdtest_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import sys,os diff --git a/test/init.sh b/test/init.sh index a379adf..8f77c82 100755 --- a/test/init.sh +++ b/test/init.sh @@ -87,8 +87,8 @@ create_test_links() { test/cmdtest.py hard test/unit_tests.py hard test/test-release.sh symbolic - test/cmdtest_py_d/common.py symbolic - test/cmdtest_py_d/ct_base.py symbolic + test/cmdtest_d/common.py symbolic + test/cmdtest_d/ct_base.py symbolic cmds/mmgen-regtest symbolic ' while read path type; do diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index b371e72..7a393df 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -42,7 +42,7 @@ init_tests() { t_lint=" - $pylint --errors-only mmgen_node_tools - $pylint --errors-only test - - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_py_d + - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_d " d_unit="low-level subsystems" From 30772b369999c6b5deae7d4ea653d46cc09680ed Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:48 +0000 Subject: [PATCH 064/108] Update for MMGen Wallet 15.1.dev6 --- MANIFEST.in | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/init.sh | 8 +++++++- test/{unit_tests_d => modtest_d}/ut_BlocksInfo.py | 0 test/{unit_tests_d => modtest_d}/ut_dep.py | 0 test/test-release.d/cfg.sh | 12 ++++++------ 7 files changed, 16 insertions(+), 10 deletions(-) rename test/{unit_tests_d => modtest_d}/ut_BlocksInfo.py (100%) rename test/{unit_tests_d => modtest_d}/ut_dep.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 1fe78fa..0443a99 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include mmgen_node_tools/data/* include test/init.sh include test/test-release.d/*.sh -include test/unit_tests_d/*.py +include test/modtest_d/*.py include test/cmdtest_d/*.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index ecc3a64..f19fa82 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev2 +3.5.dev3 diff --git a/setup.cfg b/setup.cfg index 9b82f69..edbad57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev3 + mmgen-wallet>=15.1.dev6 pyyaml yahooquery diff --git a/test/init.sh b/test/init.sh index 8f77c82..f6a6f00 100755 --- a/test/init.sh +++ b/test/init.sh @@ -77,6 +77,10 @@ create_dir_links() { done } +delete_old_stuff() { + rm -rf test/unit_tests.py +} + create_test_links() { paths=' test/include symbolic @@ -85,7 +89,7 @@ create_test_links() { test/__init__.py symbolic test/clean.py symbolic test/cmdtest.py hard - test/unit_tests.py hard + test/modtest.py hard test/test-release.sh symbolic test/cmdtest_d/common.py symbolic test/cmdtest_d/ct_base.py symbolic @@ -121,6 +125,8 @@ create_test_links() { becho 'Initializing MMGen Node Tools Test Suite' +delete_old_stuff + check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/modtest_d/ut_BlocksInfo.py similarity index 100% rename from test/unit_tests_d/ut_BlocksInfo.py rename to test/modtest_d/ut_BlocksInfo.py diff --git a/test/unit_tests_d/ut_dep.py b/test/modtest_d/ut_dep.py similarity index 100% rename from test/unit_tests_d/ut_dep.py rename to test/modtest_d/ut_dep.py diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 7a393df..a34c324 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -18,7 +18,7 @@ # mmnode-ticker OK # mmnode-txfind - -all_tests='unit lint misc scripts btc btc_rt bch_rt ltc_rt' +all_tests='mod lint misc scripts btc btc_rt bch_rt ltc_rt' groups_desc=" default - All tests minus the extra tests @@ -29,10 +29,10 @@ groups_desc=" " init_groups() { - dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + dfl_tests='mod misc scripts btc btc_rt bch_rt ltc_rt' extra_tests='lint' - noalt_tests='unit misc scripts btc btc_rt' - quick_tests='unit misc scripts btc btc_rt' + noalt_tests='mod misc scripts btc btc_rt' + quick_tests='mod misc scripts btc btc_rt' qskip_tests='lint bch_rt ltc_rt' } @@ -45,8 +45,8 @@ init_tests() { - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_d " - d_unit="low-level subsystems" - t_unit="- $unit_tests_py" + d_mod="low-level subsystems" + t_mod="- $modtest_py" d_misc="miscellaneous features" t_misc="- $cmdtest_py helpscreens" From bec7df1d6b5b3ea342fc28292ceaf83dc7498054 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 3 Jan 2025 16:42:11 +0300 Subject: [PATCH 065/108] support Nix and NixOS Quick Start for BTC: $ git clone https://github.com/mmgen/mmgen-wallet $ git clone https://github.com/mmgen/mmgen-node-tools $ cd mmgen-node-tools $ test/init.sh $ nix-shell --pure nix Enable altcoins and additional packages: # From the mmgen-node-tools repository root: $ mkdir -p ~/.mmgen $ cp ../mmgen-wallet/nix/user-packages.nix ~/.mmgen # ... edit ~/.mmgen/user-packages.nix as required ... $ nix-shell --pure nix For NixOS installation and other information, see: nix/README.node-tools ../mmgen-wallet/nix/README --- mmgen_node_tools/data/version | 2 +- nix/README.node-tools | 11 ++++++++ nix/default.nix | 6 +++++ nix/node-tools-packages.nix | 12 +++++++++ nix/shell.nix | 1 + nix/yahooquery-noversioning.patch | 45 +++++++++++++++++++++++++++++++ nix/yahooquery.nix | 44 ++++++++++++++++++++++++++++++ setup.cfg | 2 +- 8 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 nix/README.node-tools create mode 100644 nix/default.nix create mode 100644 nix/node-tools-packages.nix create mode 100644 nix/shell.nix create mode 100644 nix/yahooquery-noversioning.patch create mode 100644 nix/yahooquery.nix diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f19fa82..cf8acd2 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev3 +3.5.dev4 diff --git a/nix/README.node-tools b/nix/README.node-tools new file mode 100644 index 0000000..9308181 --- /dev/null +++ b/nix/README.node-tools @@ -0,0 +1,11 @@ +Nix configuration directory for the MMGen Node Tools suite + +Usage is as described in ‘nix/README’ in the mmgen-wallet repository, with the +following differences: + + a) all commands are executed from the repository root of mmgen-node-tools + instead of mmgen-wallet + + b) for NixOS, complete the steps as described up until the rebuild step. + Copy the contents of this directory to ‘/etc/nixos/mmgen-project’ (this + will overwrite ‘default.nix’), and continue with the rebuild step. diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..7b4b214 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,6 @@ +import ( + if builtins.pathExists ./merged-packages.nix then + ./merged-packages.nix + else + ../../mmgen-wallet/nix/merged-packages.nix + ) { add_pkgs_path = ./node-tools-packages.nix; } diff --git a/nix/node-tools-packages.nix b/nix/node-tools-packages.nix new file mode 100644 index 0000000..19792c7 --- /dev/null +++ b/nix/node-tools-packages.nix @@ -0,0 +1,12 @@ +{ pkgs, python }: + +{ + system-packages = with pkgs; { + cacert = cacert; # ticker (curl) + }; + + python-packages = with python.pkgs; { + yahooquery = (pkgs.callPackage ./yahooquery.nix {}); # ticker + pyyaml = pyyaml; # ticker + }; +} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 0000000..01991d2 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1 @@ +import ../../mmgen-wallet/nix/shell.nix { add_pkgs_path = ./node-tools-packages.nix; } diff --git a/nix/yahooquery-noversioning.patch b/nix/yahooquery-noversioning.patch new file mode 100644 index 0000000..4bd3297 --- /dev/null +++ b/nix/yahooquery-noversioning.patch @@ -0,0 +1,45 @@ +diff --git a/pyproject.toml b/pyproject.toml +index 9d3fb29..399c215 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -10,24 +10,24 @@ readme = "README.md" + + [tool.poetry.dependencies] + python = ">=3.8.1,<4.0" +-requests = "^2.31.0" +-pandas = "^2.0.3" +-requests-futures = "^1.0.1" +-tqdm = "^4.65.0" +-lxml = "^4.9.3" +-selenium = {version = "^4.10.0", optional = true} +-beautifulsoup4 = "^4.12.2" ++requests = ">=2.31.0" ++pandas = ">=2.0.3" ++requests-futures = ">=1.0.1" ++tqdm = ">=4.65.0" ++lxml = ">=4.9.3" ++selenium = {version = ">=4.10.0", optional = true} ++beautifulsoup4 = ">=4.12.2" + + [tool.poetry.dev-dependencies] +-pytest = "^7.4.0" +-isort = "^5.0.0" +-flake8 = "^6.0.0" +-mypy = "^1.4.1" +-pytest-cov = "^4.1.0" +-black = "^23.7.0" +-pre-commit = "^3.3.3" +-ipython = "^8.0.0" +-mkdocs-material = "^9.1.18" ++pytest = ">=7.4.0" ++isort = ">=5.0.0" ++flake8 = ">=6.0.0" ++mypy = ">=1.4.1" ++pytest-cov = ">=4.1.0" ++black = ">=23.7.0" ++pre-commit = ">=3.3.3" ++ipython = ">=8.0.0" ++mkdocs-material = ">=9.1.18" + + [build-system] + requires = ["poetry-core>=1.0.0"] diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix new file mode 100644 index 0000000..8b1535d --- /dev/null +++ b/nix/yahooquery.nix @@ -0,0 +1,44 @@ +{ + lib, + pkgs, + fetchFromGitHub, +}: + +with pkgs.python312.pkgs; + +buildPythonPackage rec { + pname = "yahooquery"; + version = "2.3.7"; + pyproject = true; + + disabled = pythonOlder "3.8.1"; + + src = fetchFromGitHub { + owner = "dpguthrie"; + repo = "yahooquery"; + rev = "refs/tags/v${version}"; + hash = "sha256-Iyuni1SoTB6f7nNFhN5A8Gnv9kV78frjpqvvW8qd+/M="; + }; + + patches = [ ./yahooquery-noversioning.patch ]; + + build-system = [ poetry-core ]; + + dependencies = [ + requests # ^2.31.0 + pandas # ^2.0.3 + requests-futures # ^1.0.1 + tqdm # ^4.65.0 + lxml # ^4.9.3 + selenium # {version = ^4.10.0, optional = true} + beautifulsoup4 # ^4.12.2 + ]; + + doCheck = false; # skip tests + + meta = with lib; { + description = "Python wrapper for an unofficial Yahoo Finance API"; + homepage = "https://yahooquery.dpguthrie.com"; + license = licenses.mit; + }; +} diff --git a/setup.cfg b/setup.cfg index edbad57..f63dd91 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev6 + mmgen-wallet>=15.1.dev9 pyyaml yahooquery From 7cb2fc5b08722b3a2ac1436fb9ce2d7b4fd12852 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Jan 2025 11:06:28 +0000 Subject: [PATCH 066/108] nix/shell.nix: add `repo` arg; test/init.sh: add `-c` option --- mmgen_node_tools/data/version | 2 +- nix/shell.nix | 5 ++++- setup.cfg | 2 +- test/init.sh | 17 ++++++++++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index cf8acd2..c087ce9 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev4 +3.5.dev5 diff --git a/nix/shell.nix b/nix/shell.nix index 01991d2..acfdee3 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1 +1,4 @@ -import ../../mmgen-wallet/nix/shell.nix { add_pkgs_path = ./node-tools-packages.nix; } +import ../../mmgen-wallet/nix/shell.nix { + repo = "mmgen-node-tools"; + add_pkgs_path = ./node-tools-packages.nix; +} diff --git a/setup.cfg b/setup.cfg index f63dd91..db836e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev9 + mmgen-wallet>=15.1.dev10 pyyaml yahooquery diff --git a/test/init.sh b/test/init.sh index f6a6f00..15eeddb 100755 --- a/test/init.sh +++ b/test/init.sh @@ -28,15 +28,17 @@ STDOUT_DEVNULL='>/dev/null' STDERR_DEVNULL='2>/dev/null' PROGNAME=$(basename $0) -while getopts hv OPT +while getopts hcv OPT do case "$OPT" in h) printf " %-16s Initialize the MMGen Node Tools test suite\n" "${PROGNAME}:" echo " USAGE: $PROGNAME" echo " OPTIONS: '-h' Print this help message" + echo " -c Create links from mmgen-wallet ‘cmds’ subdirectory" echo " -v Be more verbose" exit ;; v) VERBOSE=1 STDOUT_DEVNULL='' STDERR_DEVNULL='' ;; + c) CMD_LINKS=1 ;; *) exit ;; esac done @@ -123,6 +125,17 @@ create_test_links() { done <<<$paths } +create_cmd_links() { + [ "$VERBOSE" ] && becho 'Creating links to mmgen-wallet repo ‘cmds’ subdirectory' + ( + filenames=$(cd $wallet_repo/cmds && ls) + cd cmds + for filename in $filenames; do + [ -e $filename ] || ln -s "../$wallet_repo/cmds/$filename" + done + ) +} + becho 'Initializing MMGen Node Tools Test Suite' delete_old_stuff @@ -137,6 +150,8 @@ create_dir_links create_test_links +[ "$CMD_LINKS" ] && create_cmd_links + [ "$VERBOSE" ] && becho 'OK' true From b62ff991fe030448c52440f5ccf722b070ed2d41 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 6 Feb 2025 10:07:57 +0000 Subject: [PATCH 067/108] update for MMGen Wallet v15.1.dev14 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- setup.cfg | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index c087ce9..426110c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev5 +3.5.dev6 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7dd37a3..7fafac1 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -142,10 +142,10 @@ async def main(req_addrs): addr = re.match('addr\((.*?)\)',unspent['desc'])[1] addr_data[addr].append(unspent) else: - from mmgen.proto.btc.tx.base import scriptPubKey2addr + from mmgen.proto.btc.tx.base import decodeScriptPubKey for unspent in sorted(res['unspents'],key=lambda x: x['height']): - addr = scriptPubKey2addr( proto, unspent['scriptPubKey'] )[0] - addr_data[addr].append(unspent) + ds = decodeScriptPubKey(proto, unspent['scriptPubKey']) + addr_data[ds.addr].append(unspent) good_addrs = len([v for v in addr_data.values() if v]) diff --git a/setup.cfg b/setup.cfg index db836e8..e140080 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev10 + mmgen-wallet>=15.1.dev14 pyyaml yahooquery From 966c1716505f63829bb70c063560e0d3b1cff67f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 15 Mar 2025 18:23:16 +0000 Subject: [PATCH 068/108] update for MMGen Wallet v15.1.dev20 --- README.md | 2 +- mmgen_node_tools/Ticker.py | 14 ++++++++------ mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 8 ++++---- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 8 ++++---- setup.cfg | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2751da7..89c43b2 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Code repository: Code repository mirrors: [Github](https://github.com/mmgen/mmgen-node-tools) | [Gitlab](https://gitlab.com/mmgen/mmgen-node-tools) | -[Gitflic](https://gitflic.ru/project/mmgen/mmgen-node-tools) +[Codeberg](https://codeberg.org/mmgen/mmgen-node-tools) [Keybase](https://keybase.io/mmgen) | [Twitter](https://twitter.com/TheMMGenProject) | [Reddit](https://www.reddit.com/user/mmgen-py) | diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bae27c2..252ba6c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -799,7 +799,7 @@ class Ticker: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) + fmt_func = lambda t,now: time.strftime('%F %X', time.gmtime(t)) d = self.data max_w = 0 @@ -820,15 +820,17 @@ class Ticker: msg(str(e)) pass else: - t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now ) - max_w = max(len(t_fmt),max_w) + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( + (min(t,min_t) if min_t else t), + now = now) + max_w = max(len(t_fmt), max_w) self.upd_w = max_w def init_prec(self): - exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted() ) for a in self.usr_col_assets] - self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp } - self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp } + exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted()) for a in self.usr_col_assets] + self.uprec = {k: max(0, v+4) + cfg.add_prec for k, v in exp} + self.uwid = {k: 12 + max(0, abs(v)-6) + cfg.add_prec for k, v in exp} def get_id(self,asset): if asset.id: diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 426110c..aa6a660 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev6 +3.5.dev7 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7fafac1..ecf2b1a 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -60,7 +60,7 @@ def do_output(proto,addr_data,blk_hdrs): make_timestr( blk_hdrs[u['height']]['time'] ), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(color=True,iwidth=6,prec=8) + proto.coin_amt(u['amount']).fmt(6, color=True, prec=8) )) else: Msg(f'{indent}No balance') @@ -93,18 +93,18 @@ def do_output_tabular(proto,addr_data,blk_hdrs): if unspents: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(addr.view_pref, width=max_addrw, color=True), + a = addr.fmt(addr.view_pref, max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(color=True, iwidth=7, prec=8) + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8) )) else: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(addr.view_pref, width=max_addrw, color=True), + a = addr.fmt(addr.view_pref, max_addrw, color=True), u = ' -', b = '-', t = '', diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 7bee773..16591b8 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -87,7 +87,7 @@ if cfg.ignore_below: ignore_below = parse_bytespec(cfg.ignore_below) precision = ( - check_int_between(cfg.precision,min_prec,max_prec,'--precision arg') + check_int_between(cfg.precision, min_prec, max_prec, desc='--precision arg') if cfg.precision else dfl_prec ) from mmgen.term import get_terminal_size diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 1648d76..caeb178 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -171,10 +171,10 @@ async def main(): c = date(t), d = ' P' if n > nhist else '' if n < nhist else ' E', e = f'{bdr:8.5f}', - f = proto.coin_amt(sub,from_unit='satoshi').fmt(iwidth=2,prec=8), - g = proto.coin_amt(mined,from_unit='satoshi').fmt(iwidth=8,prec=8), - h = proto.coin_amt(total_mined,from_unit='satoshi').fmt(iwidth=8,prec=8) - ) for n,sub,blk,mined,total_mined,bdr,t in gen_data()) + f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), + g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), + h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) + ) for n, sub, blk, mined, total_mined, bdr, t in gen_data()) ) if cfg.list: diff --git a/setup.cfg b/setup.cfg index e140080..b43abad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev14 + mmgen-wallet>=15.1.dev20 pyyaml yahooquery From d63cb9817a249618931f526e9d5029bfe45adb0d Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 23 Mar 2025 10:12:56 +0000 Subject: [PATCH 069/108] update for MMGen Wallet v15.1.dev23 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/ct_regtest.py | 4 ++-- test/init.sh | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index aa6a660..1e3f513 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev7 +3.5.dev8 diff --git a/setup.cfg b/setup.cfg index b43abad..c67cb47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev20 + mmgen-wallet>=15.1.dev23 pyyaml yahooquery diff --git a/test/cmdtest_d/ct_regtest.py b/test/cmdtest_d/ct_regtest.py index 24f19b9..f043641 100755 --- a/test/cmdtest_d/ct_regtest.py +++ b/test/cmdtest_d/ct_regtest.py @@ -105,8 +105,8 @@ class CmdTestRegtest(CmdTestBase): ), } - def __init__(self,trunner,cfgs,spawn): - CmdTestBase.__init__(self,trunner,cfgs,spawn) + def __init__(self, cfg, trunner, cfgs, spawn): + CmdTestBase.__init__(self, cfg, trunner, cfgs, spawn) if trunner == None: return if cfg._proto.testnet: diff --git a/test/init.sh b/test/init.sh index 15eeddb..94671f9 100755 --- a/test/init.sh +++ b/test/init.sh @@ -95,6 +95,8 @@ create_test_links() { test/test-release.sh symbolic test/cmdtest_d/common.py symbolic test/cmdtest_d/ct_base.py symbolic + test/cmdtest_d/runner.py symbolic + test/cmdtest_d/group_mgr.py symbolic cmds/mmgen-regtest symbolic ' while read path type; do From 4688ee94c0dfa1a73165e72a520182e724e47c47 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 29 Mar 2025 12:12:35 +0000 Subject: [PATCH 070/108] update test suite for MMGen Wallet v15.1.dev25 --- README.md | 3 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/{ => include}/cfg.py | 2 +- test/cmdtest_d/{ct_main.py => main.py} | 6 ++-- test/cmdtest_d/{ct_misc.py => misc.py} | 6 ++-- test/cmdtest_d/{ct_regtest.py => regtest.py} | 10 +++---- test/init.sh | 31 ++++++++++++-------- 8 files changed, 34 insertions(+), 28 deletions(-) rename test/cmdtest_d/{ => include}/cfg.py (93%) rename test/cmdtest_d/{ct_main.py => main.py} (94%) rename test/cmdtest_d/{ct_misc.py => misc.py} (98%) rename test/cmdtest_d/{ct_regtest.py => regtest.py} (98%) diff --git a/README.md b/README.md index 89c43b2..a536cce 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: -Install the latest development version of [MMGen Wallet][6] for your platform: +First install the latest development version of [MMGen Wallet][6] for your +platform. Then perform the following steps: ```bash $ git clone https://github.com/mmgen/mmgen-node-tools diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1e3f513..0602083 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev8 +3.5.dev9 diff --git a/setup.cfg b/setup.cfg index c67cb47..905e659 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev23 + mmgen-wallet>=15.1.dev25 pyyaml yahooquery diff --git a/test/cmdtest_d/cfg.py b/test/cmdtest_d/include/cfg.py similarity index 93% rename from test/cmdtest_d/cfg.py rename to test/cmdtest_d/include/cfg.py index c28ce5d..5a4263d 100755 --- a/test/cmdtest_d/cfg.py +++ b/test/cmdtest_d/include/cfg.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.cfg: configuration data for cmdtest.py +test.cmdtest_d.include.cfg: configuration data for cmdtest.py """ cmd_groups_altcoin = [] diff --git a/test/cmdtest_d/ct_main.py b/test/cmdtest_d/main.py similarity index 94% rename from test/cmdtest_d/ct_main.py rename to test/cmdtest_d/main.py index 3009d2f..6e35b7d 100755 --- a/test/cmdtest_d/ct_main.py +++ b/test/cmdtest_d/main.py @@ -9,13 +9,13 @@ # https://gitlab.com/mmgen/mmgen-wallet """ -cmdtest_d.ct_main: Basic operations tests for the cmdtest.py test suite +cmdtest_d.main: Basic operations tests for the cmdtest.py test suite """ -import sys,time +import sys, time from ..include.common import cfg -from .ct_base import CmdTestBase +from .base import CmdTestBase class CmdTestMain(CmdTestBase): 'basic operations with fake RPC data' diff --git a/test/cmdtest_d/ct_misc.py b/test/cmdtest_d/misc.py similarity index 98% rename from test/cmdtest_d/ct_misc.py rename to test/cmdtest_d/misc.py index 15b109e..1a8175e 100755 --- a/test/cmdtest_d/ct_misc.py +++ b/test/cmdtest_d/misc.py @@ -9,13 +9,13 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite +test.cmdtest_d.misc: Miscellaneous test groups for the cmdtest.py test suite """ -import os,shutil +import os, shutil from ..include.common import cfg -from .ct_base import CmdTestBase +from .base import CmdTestBase refdir = os.path.join('test','ref','ticker') diff --git a/test/cmdtest_d/ct_regtest.py b/test/cmdtest_d/regtest.py similarity index 98% rename from test/cmdtest_d/ct_regtest.py rename to test/cmdtest_d/regtest.py index f043641..3b77836 100755 --- a/test/cmdtest_d/ct_regtest.py +++ b/test/cmdtest_d/regtest.py @@ -9,18 +9,18 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.ct_regtest: Regtest tests for the cmdtest.py test suite +test.cmdtest_d.regtest: Regtest tests for the cmdtest.py test suite """ -import sys,os +import sys, os from decimal import Decimal -from mmgen.util import msg_r,die,gmsg +from mmgen.util import msg_r, die, gmsg from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest -from ..include.common import cfg,imsg,stop_test_daemons,joinpath -from .ct_base import CmdTestBase +from ..include.common import cfg, imsg, stop_test_daemons, joinpath +from .base import CmdTestBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] diff --git a/test/init.sh b/test/init.sh index 94671f9..134f4f3 100755 --- a/test/init.sh +++ b/test/init.sh @@ -81,23 +81,28 @@ create_dir_links() { delete_old_stuff() { rm -rf test/unit_tests.py + rm -rf test/cmdtest_d/common.py + rm -rf test/cmdtest_d/ct_base.py + rm -rf test/cmdtest_d/group_mgr.py + rm -rf test/cmdtest_d/runner.py } create_test_links() { paths=' - test/include symbolic - test/overlay/__init__.py symbolic - test/overlay/fakemods/mmgen symbolic - test/__init__.py symbolic - test/clean.py symbolic - test/cmdtest.py hard - test/modtest.py hard - test/test-release.sh symbolic - test/cmdtest_d/common.py symbolic - test/cmdtest_d/ct_base.py symbolic - test/cmdtest_d/runner.py symbolic - test/cmdtest_d/group_mgr.py symbolic - cmds/mmgen-regtest symbolic + test/include symbolic + test/overlay/__init__.py symbolic + test/overlay/fakemods/mmgen symbolic + test/__init__.py symbolic + test/clean.py symbolic + test/cmdtest.py hard + test/modtest.py hard + test/test-release.sh symbolic + test/cmdtest_d/base.py symbolic + test/cmdtest_d/include/common.py symbolic + test/cmdtest_d/include/runner.py symbolic + test/cmdtest_d/include/group_mgr.py symbolic + test/cmdtest_d/include/pexpect.py symbolic + cmds/mmgen-regtest symbolic ' while read path type; do [ "$path" ] || continue From ab93f54d541d4ee6511f9bf37e703d50fc2745f5 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 30 Mar 2025 10:18:52 +0000 Subject: [PATCH 071/108] update test suite for MMGen Wallet v15.1.dev26 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0602083..de6f13c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev9 +3.5.dev10 diff --git a/setup.cfg b/setup.cfg index 905e659..0ab2734 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev25 + mmgen-wallet>=15.1.dev26 pyyaml yahooquery From 106f201300d7dadb3a70e979db9373e9f6ca0e34 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 24 May 2025 10:10:24 +0000 Subject: [PATCH 072/108] update for MMGen Wallet v15.1.dev39 --- MANIFEST.in | 1 + mmgen_node_tools/BlocksInfo.py | 2 +- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 2 +- setup.cfg | 3 +-- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0443a99..77b53ac 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,5 +6,6 @@ include test/init.sh include test/test-release.d/*.sh include test/modtest_d/*.py include test/cmdtest_d/*.py +include test/cmdtest_d/include/cfg.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 26e6cfa..f19089a 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -26,7 +26,7 @@ from time import strftime,gmtime from decimal import Decimal from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int -from mmgen.rpc import json_encoder +from mmgen.rpc.util import json_encoder class RangeParser: diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index de6f13c..70beee3 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev10 +3.5.dev11 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 16591b8..e7b9952 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -103,7 +103,7 @@ class fee_bracket: def log(data,fn): import json - from mmgen.rpc import json_encoder + from mmgen.rpc.util import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( cfg = cfg, diff --git a/setup.cfg b/setup.cfg index 0ab2734..cd27742 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ project_urls = Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = - License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows Operating System :: MacOS @@ -38,7 +37,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev26 + mmgen-wallet>=15.1.dev39 pyyaml yahooquery From 5a24cbfc0a50d03203710c387763a1b893f3d5f3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 24 Sep 2025 10:45:50 +0000 Subject: [PATCH 073/108] Version 3.5.0 - released concurrently with MMGen Wallet v16.0.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 70beee3..1545d96 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev11 +3.5.0 diff --git a/setup.cfg b/setup.cfg index cd27742..9bca61c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev39 + mmgen-wallet==16.0.0 pyyaml yahooquery From cab8be016707b6886cbacf56666611b9a3fdad98 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:37 +0000 Subject: [PATCH 074/108] pyproject.toml: add ruff data --- pyproject.toml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 81515d2..25af44f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,41 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.ruff] +line-length = 106 +indent-width = 4 + +[tool.ruff.format] +quote-style = "single" +indent-style = "tab" + +[tool.ruff.lint] +ignore = [ + "E401", # multiple imports per line + "E701", # multiple statements per line + "E721", # use isinstance() + "E731", # lambda instead of def + "E402", # module import not top of file + "E722", # bare except + "E713", # membership 'not in' + "E741", # ambiguous variable name +] + +[tool.ruff.lint.per-file-ignores] +"test/include/common.py" = [ "F821" ] # undefined name 'cfg' +"test/misc/input_func.py" = [ "F401" ] # imported but unused +"test/modtest_d/cashaddr.py" = [ "F841" ] # assigned to but never used +"test/modtest_d/dep.py" = [ "F401" ] # imported but unused +"test/modtest_d/testdep.py" = [ "F401" ] # imported but unused +"test/modtest_d/obj.py" = [ "F841" ] # assigned to but never used +"test/objtest_d/*" = [ "F401" ] # imported but unused +"test/objattrtest_d/*" = [ "F401" ] # imported but unused +"test/overlay/fakemods/*" = [ "F403", "F405" ] # `import *` used +"test/*.py" = [ "F401" ] # imported but unused +"test/colortest.py" = [ "F403", "F405" ] # `import *` used +"test/tooltest2.py" = [ "F403", "F405" ] # `import *` used +"test/overlay/tree/*" = [ "ALL" ] + [tool.pylint.format] indent-string = "\t" indent-after-paren = 2 From 5c9d7053017f463193b9b5aa9dfb036f1844d697 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 075/108] test suite: lint, whitespace --- test/cmdtest_d/main.py | 2 +- test/cmdtest_d/misc.py | 4 ++-- test/cmdtest_d/regtest.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/cmdtest_d/main.py b/test/cmdtest_d/main.py index 6e35b7d..f1881f8 100755 --- a/test/cmdtest_d/main.py +++ b/test/cmdtest_d/main.py @@ -41,7 +41,7 @@ class CmdTestMain(CmdTestBase): def peerblocks(self,args,expect_list=None,pexpect_spawn=False): t = self.spawn( - f'mmnode-peerblocks', + 'mmnode-peerblocks', args, pexpect_spawn = pexpect_spawn ) if cfg.exact_output: # disable echoing of input diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 1a8175e..8a39462 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -32,7 +32,7 @@ class CmdTestHelp(CmdTestBase): color = True def version(self): - t = self.spawn(f'mmnode-netrate',['--version']) + t = self.spawn('mmnode-netrate', ['--version']) t.expect('MMNODE-NETRATE version') return t @@ -108,7 +108,7 @@ class CmdTestScripts(CmdTestBase): def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): t = self.spawn( - f'mmnode-ticker', + 'mmnode-ticker', (['--cached-data'] if cached else []) + self.ticker_args + args, exit_val = exit_val) if expect_list: diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index 3b77836..4792dee 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -107,7 +107,7 @@ class CmdTestRegtest(CmdTestBase): def __init__(self, cfg, trunner, cfgs, spawn): CmdTestBase.__init__(self, cfg, trunner, cfgs, spawn) - if trunner == None: + if trunner is None: return if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') @@ -328,7 +328,7 @@ class CmdTestRegtest(CmdTestBase): async def do_tx(inputs,outputs,wif): tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) - assert tx['complete'] == True + assert tx['complete'] return tx['hex'] async def do_tx1(): @@ -371,13 +371,13 @@ class CmdTestRegtest(CmdTestBase): imsg(f'Creating funding transaction with {nTxs} outputs of value {tx1_amt} {self.proto.coin}') tx1_hex = await do_tx1() - imsg(f'Relaying funding transaction') + imsg('Relaying funding transaction') await r.rpc_call('sendrawtransaction',tx1_hex) - imsg(f'Mining a block') + imsg('Mining a block') await r.generate(1,silent=True) - imsg(f'Generating fees for mempool transactions') + imsg('Generating fees for mempool transactions') fees = list(gen_fees(nTxs,2,120)) imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') From 4d1f4577a7758c0c028166d7f59538ec53d74d8d Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 076/108] lint, whitespace --- mmgen_node_tools/BlocksInfo.py | 2 +- mmgen_node_tools/PollDisplay.py | 2 +- mmgen_node_tools/Sound.py | 2 +- mmgen_node_tools/Ticker.py | 23 ++++++++++++--------- mmgen_node_tools/Util.py | 1 - mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 5 ++--- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index f19089a..d5eac9e 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -361,7 +361,7 @@ class BlocksInfo: if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) if nblocks: - if first == None: + if first is None: first = self.tip - nblocks + 1 last = first + nblocks - 1 diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 72bedac..1de8a1a 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -52,7 +52,7 @@ class PollDisplay: count += 1 async def process_input(): - if self.input == None: + if self.input is None: sys.exit(1) elif self.input == 'q': msg('') diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index 88d3b02..df9f8ae 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -33,7 +33,7 @@ def timespec2secs(ts): mul = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24 } pat = r'^([0-9]+)([smhd]*)$' m = re.match(pat,ts) - if m == None: + if m is None: die(2,"'%s': invalid time specifier" % ts) a,b = m.groups() return int(a) * (mul[b] if b else 1) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 252ba6c..6c8bc08 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -19,13 +19,13 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,re,time,datetime,json,yaml,random -from subprocess import run,PIPE,CalledProcessError +import sys, os, re, time, datetime, json, yaml, random +from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red,yellow,green,blue,orange,gray -from mmgen.util import msg,msg_r,Msg,Msg_r,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.color import red, yellow, green, blue, orange, gray +from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -89,9 +89,11 @@ class DataSource: msg('') from .Misc import curl_exit_codes msg(red(curl_exit_codes[e.returncode])) - msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) + msg(red('Command line:\n {}'.format( + ' '.join((repr(i) if ' ' in i else i) for i in e.cmd)))) from mmgen.exception import MMGenCalledProcessError - raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') + raise MMGenCalledProcessError( + f'Subprocess returned non-zero exit status {e.returncode}') def get_data(self): @@ -327,15 +329,16 @@ class DataSource: def postprocess_data(self,data): def gen(): keys = set() - for key,val in data.items(): + d = {} + for key, val in data.items(): if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) if (sym := m[1]) in keys: d[date] = val else: keys.add(sym) - d = {date:val} - yield (sym,d) + d = {date: val} + yield (sym, d) return dict(gen()) def assets_list_gen(cfg_in): @@ -936,7 +939,7 @@ class Ticker: def fmt_row(self,d,amt=None,amt_fmt=None): def fmt_pct(n): - return gray(' --') if n == None else (red,green)[n>=0](f'{n:+7.2f}') + return gray(' --') if n is None else (red,green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 9060f68..1d5426b 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -20,7 +20,6 @@ mmgen_node_tools.Util: utility functions for MMGen node tools """ import time -from mmgen.util import suf def get_hms(t=None,utc=False,no_secs=False): secs = t or time.time() diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index ecf2b1a..747d0bb 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -14,7 +14,7 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain import sys -from mmgen.obj import CoinTxID,Int +from mmgen.obj import CoinTxID from mmgen.cfg import Config from mmgen.util import msg,Msg,die,suf,make_timestr,async_run from mmgen.color import red diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index caeb178..50916ab 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -21,7 +21,6 @@ mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) """ import time -from decimal import Decimal from mmgen.cfg import Config from mmgen.util import async_run @@ -148,8 +147,8 @@ async def main(): b = 'BLOCK', c = 'DATE', d = '', - e = f'BDI (mins)', - f = f'SUBSIDY ({proto.coin})', + e = 'BDI (mins)', + f = 'SUBSIDY ({proto.coin})', g = f'MINED ({proto.coin})', h = f'TOTAL MINED ({proto.coin})' ) From bfb2dd839aab1d221e38d6e6532dca20f2904632 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 077/108] setup.cfg: bump requirements --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9bca61c..9af335b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,11 +33,11 @@ classifiers = Development Status :: 5 - Production/Stable [options] -python_requires = >=3.9 +python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet==16.0.0 + mmgen-wallet>=16.0.0 pyyaml yahooquery From 37c74e361c0f695a43b3f08b4b85cc2106b92665 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 078/108] use match statement where practicable --- mmgen_node_tools/BlocksInfo.py | 102 ++++++++++++++++++--------------- mmgen_node_tools/Ticker.py | 21 +++---- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index d5eac9e..42ccc23 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -197,18 +197,18 @@ class BlocksInfo: return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') def parse_cmd_args(): # => (block_list, first, last, step) - if not cmd_args: - return (None,self.tip,self.tip,None) - elif len(cmd_args) == 1: - r = self.parse_rangespec(cmd_args[0]) - return ( - list(range(r.first,r.last+1,r.step)) if r.step else None, - r.first, - r.last, - r.step - ) - else: - return ([self.conv_blkspec(a) for a in cmd_args],None,None,None) + match cmd_args: + case [] | None: + return (None, self.tip, self.tip, None) + case [arg]: + r = self.parse_rangespec(arg) + return ( + list(range(r.first, r.last+1, r.step)) if r.step else None, + r.first, + r.last, + r.step) + case [*args]: + return ([self.conv_blkspec(a) for a in args], None, None, None) self.cfg = cfg self.rpc = rpc @@ -323,25 +323,29 @@ class BlocksInfo: repl = (name if add_name else '') + ':' + (fill_char if name in fill else '') yield (ls + self.fields[name].fs.replace(':',repl) + rs) - def conv_blkspec(self,arg): - if str(arg).lower() == 'cur': - return self.tip - elif is_int(arg): - if int(arg) < 0: - die(1,f'{arg}: block number must be non-negative') - elif int(arg) > self.tip: - die(1,f'{arg}: requested block height greater than current chain tip!') - else: - return int(arg) - else: - die(1,f'{arg}: invalid block specifier') + def conv_blkspec(self, arg): + match arg: + case str() if arg.lower() == 'cur': + return self.tip + case x if is_int(x): + match int(arg): + case x if x < 0: + die(1, f'{x}: block number must be non-negative') + case x if x > self.tip: + die(1, f'{x}: requested block height greater than current chain tip!') + case x: + return x + case _: + die(1, f'{arg}: invalid block specifier') - def check_nblocks(self,arg): - if arg <= 0: - die(1,'nBlocks must be a positive integer') - elif arg > self.tip: - die(1, f"'{arg}': nBlocks must be less than current chain height") - return arg + def check_nblocks(self, arg): + match arg: + case x if x <= 0: + die(1, 'nBlocks must be a positive integer') + case x if x > self.tip: + die(1, f'{arg}: nBlocks must be less than current chain height') + case _: + return arg def parse_rangespec(self,arg): @@ -487,20 +491,21 @@ class BlocksInfo: def fmt_stat_item(self,fs,s): return fs.format(s) if type(fs) == str else fs(s) - async def output_stats(self,res,sname): + async def output_stats(self, res, sname): def gen(data): for d in data: - if len(d) == 2: - yield (indent+d[0]).format(**{k:self.fmt_stat_item(*v) for k,v in d[1].items()}) - elif len(d) == 4: - yield (indent+d[0]).format(self.fmt_stat_item(d[2],d[3])) - elif type(d) == str: - yield d - else: - assert False, f'{d}: invalid stats data' + match d: + case [a, b]: + yield (indent + a).format(**{k:self.fmt_stat_item(*v) for k, v in b.items()}) + case [a, _, b, c]: + yield (indent + a).format(self.fmt_stat_item(b, c)) + case str(): + yield d + case _: + assert False, f'{d}: invalid stats data' - foo,data = await res + foo, data = await res indent = '' if sname in self.noindent_stats else ' ' Msg('\n'.join(gen(data))) @@ -755,13 +760,16 @@ class JSONBlocksInfo(BlocksInfo): def gen(data): for d in data: - if len(d) == 2: - for k,v in d[1].items(): - yield (k,self.fmt_stat_item(*v)) - elif len(d) == 4: - yield (d[1],self.fmt_stat_item(d[2],d[3])) - elif type(d) != str: - assert False, f'{d}: invalid stats data' + match d: + case [_, a]: + for k, v in a.items(): + yield (k, self.fmt_stat_item(*v)) + case [_, a, b, c]: + yield (a, self.fmt_stat_item(b, c)) + case str(): + pass + case _: + assert False, f'{d}: invalid stats data' varname,data = await res Msg_r(', "{}_data": {}'.format( varname, json.dumps(dict(gen(data)),cls=json_encoder) )) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6c8bc08..95e03f2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -125,16 +125,17 @@ class DataSource: else: die(1,self.rate_limit_errmsg(elapsed)) - if data_type == 'json': - try: - data = json.loads(data_in) - except: - self.json_data_error_msg(data_in) - die(2,'Retrieved data is not valid JSON, exiting') - json_text = data_in - elif data_type == 'python': - data = data_in - json_text = json.dumps(data_in) + match data_type: + case 'json': + try: + data = json.loads(data_in) + except: + self.json_data_error_msg(data_in) + die(2,'Retrieved data is not valid JSON, exiting') + json_text = data_in + case 'python': + data = data_in + json_text = json.dumps(data_in) if not data: if use_cached_data: From fe45fbaa239ae6c28ed056cd1c2f0977550bad45 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 079/108] minor cleanups --- mmgen_node_tools/main_blocks_info.py | 4 ++-- mmgen_node_tools/main_halving_calculator.py | 4 ++-- test/modtest_d/__init__.py | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100755 test/modtest_d/__init__.py diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index c2bbf59..18136c4 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -164,7 +164,7 @@ async def main(): cls = JSONBlocksInfo if cfg.json else BlocksInfo - m = cls( cfg, cfg._args, await rpc_init(cfg,ignore_wallet=True) ) + m = cls(cfg, cfg._args, await rpc_init(cfg, ignore_wallet=True)) if m.fnames and not cfg.no_header: m.print_header() @@ -178,4 +178,4 @@ async def main(): m.finalize_output() -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 50916ab..8f0a3a3 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -67,7 +67,7 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init( cfg, proto, ignore_wallet=True ) + c = await rpc_init(cfg, proto, ignore_wallet=True) tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' @@ -181,4 +181,4 @@ async def main(): else: print_current_stats() -async_run(main()) +async_run(cfg, main) diff --git a/test/modtest_d/__init__.py b/test/modtest_d/__init__.py new file mode 100755 index 0000000..86190b5 --- /dev/null +++ b/test/modtest_d/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +""" +test.modtest_d: shared data for module tests for the MMGen Node Tools suite +""" + +altcoin_tests = [] From 1f12baac4c246f812942e4817dfd8541d5ee8913 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 080/108] update for MMGen Wallet v16.1.dev3 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_netrate.py | 2 +- mmgen_node_tools/main_peerblocks.py | 8 ++++---- mmgen_node_tools/main_txfind.py | 2 +- setup.cfg | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1545d96..229c4d8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.0 +3.6.dev0 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 747d0bb..37f0482 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -167,6 +167,6 @@ if len(cfg._args) < 1: die(1,'This command requires at least one coin address argument') try: - async_run(main(cfg._args)) + async_run(cfg, main, args=[cfg._args]) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index e7b9952..3211f7f 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -220,4 +220,4 @@ async def main(): await c.call('getblockcount') )) + '\n\n' + '\n'.join(gen_body(data)) + '\n' ) -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index 52b509d..b273ac6 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -67,6 +67,6 @@ async def main(): rs,ss,ts = (r,s,t) try: - async_run(main()) + async_run(cfg, main) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 6ebb5a3..35292ea 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -31,10 +31,10 @@ opts_data = { } } -async def main(): +from mmgen.cfg import Config +cfg = Config(opts_data=opts_data) - from mmgen.cfg import Config - cfg = Config(opts_data=opts_data) +async def main(): from mmgen.rpc import rpc_init rpc = await rpc_init(cfg,ignore_wallet=True) @@ -48,4 +48,4 @@ async def main(): await peers.run(rpc) from mmgen.util import async_run -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 4a54430..2411f36 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -92,4 +92,4 @@ msgs = msg_data['quiet' if cfg.quiet else 'normal'] if len(cfg._args) != 1: die(1,'One transaction ID must be specified') -sys.exit(async_run(main(cfg._args[0]))) +sys.exit(async_run(cfg, main, args=[cfg._args[0]])) diff --git a/setup.cfg b/setup.cfg index 9af335b..29481b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.0.0 + mmgen-wallet>=16.1.dev3 pyyaml yahooquery From 0bad23b77bd2327937ec82ef7e8d4dba127e44be Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 3 Oct 2025 10:31:49 +0000 Subject: [PATCH 081/108] update for MMGen Wallet v16.1.dev4 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/include/cfg.py | 12 ++++++++---- test/cmdtest_d/misc.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 229c4d8..9f32e9d 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev0 +3.6.dev1 diff --git a/setup.cfg b/setup.cfg index 29481b2..d48a8c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev3 + mmgen-wallet>=16.1.dev4 pyyaml yahooquery diff --git a/test/cmdtest_d/include/cfg.py b/test/cmdtest_d/include/cfg.py index 5a4263d..08b6263 100755 --- a/test/cmdtest_d/include/cfg.py +++ b/test/cmdtest_d/include/cfg.py @@ -12,13 +12,17 @@ test.cmdtest_d.include.cfg: configuration data for cmdtest.py """ +from collections import namedtuple + cmd_groups_altcoin = [] +gd = namedtuple('cmd_groups_data', ['clsname', 'params']) + cmd_groups_dfl = { - 'main': ('CmdTestMain',{}), - 'helpscreens': ('CmdTestHelp',{'modname':'misc','full_data':True}), - 'scripts': ('CmdTestScripts',{'modname':'misc'}), - 'regtest': ('CmdTestRegtest',{}), + 'main': gd('CmdTestMain', {}), + 'helpscreens': gd('CmdTestHelp', {'modname': 'misc', 'full_data': True}), + 'scripts': gd('CmdTestScripts', {'modname': 'misc'}), + 'regtest': gd('CmdTestRegtest', {}), } cmd_groups_extra = {} diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 8a39462..9528b76 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -125,7 +125,7 @@ class CmdTestScripts(CmdTestBase): if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - ret = t.expect(['proxy host could not be resolved', 'ProxyError']) + ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) t.exit_val = 1 if ret else 3 return t From 0b79ef719b3712b64571b63dbd3177ab247aa8b3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 09:56:55 +0000 Subject: [PATCH 082/108] whitespace, minor changes (16 files) --- mmgen_node_tools/BlocksInfo.py | 402 +++++++-------- mmgen_node_tools/PeerBlocks.py | 55 +- mmgen_node_tools/PollDisplay.py | 12 +- mmgen_node_tools/Sound.py | 32 +- mmgen_node_tools/Ticker.py | 485 ++++++++---------- mmgen_node_tools/Util.py | 60 ++- mmgen_node_tools/main_addrbal.py | 66 ++- mmgen_node_tools/main_blocks_info.py | 19 +- mmgen_node_tools/main_feeview.py | 65 ++- mmgen_node_tools/main_halving_calculator.py | 56 +- mmgen_node_tools/main_netrate.py | 21 +- mmgen_node_tools/main_peerblocks.py | 7 +- mmgen_node_tools/main_ticker.py | 23 +- mmgen_node_tools/main_txfind.py | 17 +- test/cmdtest_d/regtest.py | 8 +- .../fakemods/mmgen_node_tools/PeerBlocks.py | 8 +- 16 files changed, 629 insertions(+), 707 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 42ccc23..a59bf50 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -20,58 +20,57 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of blocks """ -import re,json +import re, json from collections import namedtuple -from time import strftime,gmtime +from time import strftime, gmtime from decimal import Decimal -from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int +from mmgen.util import msg, Msg, Msg_r, die, suf, secs_to_ms, secs_to_dhms, is_int from mmgen.rpc.util import json_encoder class RangeParser: debug = False - def __init__(self,caller,arg): + def __init__(self, caller, arg): self.caller = caller self.arg = self.orig_arg = arg - def parse(self,target): - ret = getattr(self,'parse_'+target)() + def parse(self, target): + ret = getattr(self, 'parse_'+target)() if self.debug: msg(f'arg after parse({target}): {self.arg}') return ret def finalize(self): if self.arg: - die(1,f'{self.orig_arg!r}: invalid range specifier') + die(1, f'{self.orig_arg!r}: invalid range specifier') def parse_from_tip(self): - m = re.match(r'-([0-9]+)(.*)',self.arg) + m = re.match(r'-([0-9]+)(.*)', self.arg) if m: - res,self.arg = (m[1],m[2]) + res, self.arg = (m[1], m[2]) return self.caller.check_nblocks(int(res)) def parse_abs_range(self): - m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',self.arg) + m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)', self.arg) if m: if self.debug: msg(f'abs_range parse: first={m[1]}, last={m[3]}') self.arg = m[4] return ( self.caller.conv_blkspec(m[1]), - self.caller.conv_blkspec(m[3]) if m[3] else None - ) - return (None,None) + self.caller.conv_blkspec(m[3]) if m[3] else None) + return (None, None) def parse_add(self): - m = re.match(r'\+([0-9*]+)(.*)',self.arg) + m = re.match(r'\+([0-9*]+)(.*)', self.arg) if m: - res,self.arg = (m[1],m[2]) + res, self.arg = (m[1], m[2]) if res.strip('*') != res: - die(1,f"'+{res}': malformed nBlocks specifier") + die(1, f"'+{res}': malformed nBlocks specifier") if len(res) > 30: - die(1,f"'+{res}': overly long nBlocks specifier") + die(1, f"'+{res}': overly long nBlocks specifier") return self.caller.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe class BlocksInfo: @@ -81,33 +80,33 @@ class BlocksInfo: total_solve_time = 0 header_printed = False - bf = namedtuple('block_info_fields',['fmt_func','src','fs','hdr1','hdr2','key1','key2']) + bf = namedtuple('block_info_fields', ['fmt_func', 'src', 'fs', 'hdr1', 'hdr2', 'key1', 'key2']) # bh=getblockheader, bs=getblockstats, lo=local fields = { - 'block': bf( None, 'bh', '{:<6}', '', 'Block', 'height', None ), - 'hash': bf( None, 'bh', '{:<64}', '', 'Hash', 'hash', None ), - 'date': bf( 'da', 'bh', '{:<19}', '', 'Date', 'time', None ), - 'interval': bf( 'td', 'lo', '{:>8}', 'Solve', 'Time ', 'interval', None ), - 'subsidy': bf( 'su', 'bs', '{:<5}', 'Sub-', 'sidy', 'subsidy', None ), - 'totalfee': bf( 'tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None ), - 'size': bf( None, 'bs', '{:>7}', '', 'Size', 'total_size', None ), - 'weight': bf( None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None ), - 'fee90': bf( 'fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4 ), - 'fee75': bf( 'fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3 ), - 'fee50': bf( 'fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2 ), - 'fee25': bf( 'fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1 ), - 'fee10': bf( 'fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0 ), - 'fee_max': bf( 'fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None ), - 'fee_avg': bf( 'fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None ), - 'fee_min': bf( 'fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None ), - 'nTx': bf( None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None ), - 'inputs': bf( None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None ), - 'outputs': bf( None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None ), - 'utxo_inc': bf( None, 'bs', '{:>6}', ' UTXO', ' Incr', 'utxo_increase', None ), - 'version': bf( None, 'bh', '{:<8}', '', 'Version', 'versionHex', None ), - 'difficulty': bf( 'di', 'bh', '{:<8}', 'Diffi-','culty', 'difficulty', None ), - 'miner': bf( None, 'lo', '{:<5}', '', 'Miner', 'miner', None ), - } + 'block': bf(None, 'bh', '{:<6}', '', 'Block', 'height', None), + 'hash': bf(None, 'bh', '{:<64}', '', 'Hash', 'hash', None), + 'date': bf('da', 'bh', '{:<19}', '', 'Date', 'time', None), + 'interval': bf('td', 'lo', '{:>8}', 'Solve', 'Time ', 'interval', None), + 'subsidy': bf('su', 'bs', '{:<5}', 'Sub-', 'sidy', 'subsidy', None), + 'totalfee': bf('tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None), + 'size': bf(None, 'bs', '{:>7}', '', 'Size', 'total_size', None), + 'weight': bf(None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None), + 'fee90': bf('fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4), + 'fee75': bf('fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3), + 'fee50': bf('fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2), + 'fee25': bf('fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1), + 'fee10': bf('fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0), + 'fee_max': bf('fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None), + 'fee_avg': bf('fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None), + 'fee_min': bf('fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None), + 'nTx': bf(None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None), + 'inputs': bf(None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None), + 'outputs': bf(None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None), + 'utxo_inc': bf(None, 'bs', '{:>6}', ' UTXO', ' Incr', 'utxo_increase', None), + 'version': bf(None, 'bh', '{:<8}', '', 'Version', 'versionHex', None), + 'difficulty': bf('di', 'bh', '{:<8}', 'Diffi-','culty', 'difficulty', None), + 'miner': bf(None, 'lo', '{:<5}', '', 'Miner', 'miner', None)} + dfl_fields = ( 'block', 'date', @@ -121,8 +120,8 @@ class BlocksInfo: 'fee10', 'fee_avg', 'fee_min', - 'version', - ) + 'version') + fixed_fields = ( 'block', # until ≈ 09/01/2028 (block 1000000) 'hash', @@ -131,36 +130,34 @@ class BlocksInfo: 'weight', # until ≈ 2.5x block size increase 'version', 'subsidy', # until ≈ 01/04/2028 (increases by 1 digit per halving until 9th halving [max 10 digits]) - 'difficulty', # until 1.00e+100 (i.e. never) - ) + 'difficulty') # until 1.00e+100 (i.e. never) # column width adjustment data: - fs_lsqueeze = ('totalfee','inputs','outputs','nTx') + fs_lsqueeze = ('totalfee', 'inputs', 'outputs', 'nTx') fs_rsqueeze = () fs_groups = ( - ('fee10','fee25','fee50','fee75','fee90','fee_avg','fee_min','fee_max'), - ) + ('fee10', 'fee25', 'fee50', 'fee75', 'fee90', 'fee_avg', 'fee_min', 'fee_max')) fs_lsqueeze2 = ('interval',) - all_stats = ['col_avg','range','avg','mini_avg','total','diff'] - dfl_stats = ['range','mini_avg','diff'] + all_stats = ['col_avg', 'range', 'avg', 'mini_avg', 'total', 'diff'] + dfl_stats = ['range', 'mini_avg', 'diff'] noindent_stats = ['col_avg'] - avg_stats_skip = {'block', 'hash', 'date', 'version','miner'} + avg_stats_skip = {'block', 'hash', 'date', 'version', 'miner'} - range_data = namedtuple('parsed_range_data',['first','last','from_tip','nblocks','step']) + range_data = namedtuple('parsed_range_data', ['first', 'last', 'from_tip', 'nblocks', 'step']) - t_fmt = lambda self,t: f'{t/86400:.2f} days' if t > 172800 else f'{t/3600:.2f} hrs' + t_fmt = lambda self, t: f'{t/86400:.2f} days' if t > 172800 else f'{t/3600:.2f} hrs' @classmethod - def parse_cslist(cls,uarg,full_set,dfl_set,desc): + def parse_cslist(cls, uarg, full_set, dfl_set, desc): - def make_list(m,func): + def make_list(m, func): groups_lc = [set(e.lower() for e in gi.split(',')) for gi in m.groups()] for group in groups_lc: for e in group: if e not in full_set_lc: - die(1,f'{e!r}: unrecognized {desc}') + die(1, f'{e!r}: unrecognized {desc}') # display elements in order: return [e for e in full_set if e.lower() in func(groups_lc)] @@ -168,33 +165,31 @@ class BlocksInfo: dfl_set_lc = set(e.lower() for e in dfl_set) cspat = r'(\w+(?:,\w+)*)' - for pat,func in ( - ( rf'{cspat}$', lambda g: g[0] ), - ( rf'\+{cspat}$', lambda g: dfl_set_lc | g[0] ), - ( rf'\-{cspat}$', lambda g: dfl_set_lc - g[0] ), - ( rf'\+{cspat}\-{cspat}$', lambda g: ( dfl_set_lc | g[0] ) - g[1] ), - ( rf'\-{cspat}\+{cspat}$', lambda g: ( dfl_set_lc - g[0] ) | g[1] ), - ( rf'all\-{cspat}$', lambda g: full_set_lc - g[0] ) - ): - m = re.match(pat,uarg,re.ASCII|re.IGNORECASE) + for pat, func in ( + (rf'{cspat}$', lambda g: g[0]), + (rf'\+{cspat}$', lambda g: dfl_set_lc | g[0]), + (rf'\-{cspat}$', lambda g: dfl_set_lc - g[0]), + (rf'\+{cspat}\-{cspat}$', lambda g: (dfl_set_lc | g[0]) - g[1]), + (rf'\-{cspat}\+{cspat}$', lambda g: (dfl_set_lc - g[0]) | g[1]), + (rf'all\-{cspat}$', lambda g: full_set_lc - g[0])): + m = re.match(pat, uarg, re.ASCII|re.IGNORECASE) if m: - return make_list(m,func) + return make_list(m, func) else: - die(1,f'{uarg}: invalid parameter') + die(1, f'{uarg}: invalid parameter') - def __init__(self,cfg,cmd_args,rpc): + def __init__(self, cfg, cmd_args, rpc): - def parse_cs_uarg(uarg,full_set,dfl_set,desc): + def parse_cs_uarg(uarg, full_set, dfl_set, desc): return ( full_set if uarg == 'all' else [] if uarg == 'none' else - self.parse_cslist(uarg,full_set,dfl_set,desc) - ) + self.parse_cslist(uarg, full_set, dfl_set, desc)) def get_fields(): - return parse_cs_uarg(self.cfg.fields,list(self.fields),self.dfl_fields,'field') + return parse_cs_uarg(self.cfg.fields, list(self.fields), self.dfl_fields, 'field') def get_stats(): - return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') + return parse_cs_uarg(self.cfg.stats.lower(), self.all_stats, self.dfl_stats, 'stat') def parse_cmd_args(): # => (block_list, first, last, step) match cmd_args: @@ -217,7 +212,7 @@ class BlocksInfo: from_satoshi = self.rpc.proto.coin_amt.satoshi to_satoshi = 1 / from_satoshi - self.block_list,self.first,self.last,self.step = parse_cmd_args() + self.block_list, self.first, self.last, self.step = parse_cmd_args() have_segwit = self.rpc.info('segwit_is_active') @@ -228,35 +223,33 @@ class BlocksInfo: self.stats_deps = { 'avg': set(self.fields) - self.avg_stats_skip, 'col_avg': set(self.fields) - self.avg_stats_skip, - 'mini_avg': {'interval','size'} | ({'weight'} if have_segwit else set()), - 'total': {'interval','subsidy','totalfee','nTx','inputs','outputs','utxo_inc'}, + 'mini_avg': {'interval', 'size'} | ({'weight'} if have_segwit else set()), + 'total': {'interval', 'subsidy', 'totalfee', 'nTx', 'inputs', 'outputs', 'utxo_inc'}, 'range': {}, - 'diff': {}, - } + 'diff': {}} self.fmt_funcs = { - 'da': lambda arg: strftime('%Y-%m-%d %X',gmtime(arg)), + 'da': lambda arg: strftime('%Y-%m-%d %X', gmtime(arg)), 'td': lambda arg: ( - '-{:02}:{:02}'.format(abs(arg)//60,abs(arg)%60) if arg < 0 else - ' {:02}:{:02}'.format(arg//60,arg%60) ), + '-{:02}:{:02}'.format(abs(arg)//60, abs(arg)%60) if arg < 0 else + ' {:02}:{:02}'.format(arg//60, arg%60)), 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), 'fe': lambda arg: str(arg), - 'di': lambda arg: '{:.2e}'.format(Decimal(arg)), - } + 'di': lambda arg: '{:.2e}'.format(Decimal(arg))} if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), 'fe': lambda arg: str(int(Decimal(arg) * to_satoshi)), - 'tf': lambda arg: '{:.8f}'.format(Decimal(arg)), - }) + 'tf': lambda arg: '{:.8f}'.format(Decimal(arg))}) self.fnames = tuple( - [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] if self.cfg.header_info else - get_fields() if self.cfg.fields else - self.dfl_fields - ) + [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] + if self.cfg.header_info + else get_fields() if self.cfg.fields + else self.dfl_fields) + if self.cfg.miner_info and 'miner' not in self.fnames: self.fnames += ('miner',) @@ -266,15 +259,15 @@ class BlocksInfo: if 'diff' in self.stats and not self.cfg.stats and self.last != self.tip: self.stats.remove('diff') - if {'avg','col_avg'} <= set(self.stats) and self.cfg.stats_only: + if {'avg', 'col_avg'} <= set(self.stats) and self.cfg.stats_only: self.stats.remove('col_avg') - if {'avg','mini_avg'} <= set(self.stats): + if {'avg', 'mini_avg'} <= set(self.stats): self.stats.remove('mini_avg') if self.cfg.full_stats: add_fnames = {fname for sname in self.stats for fname in self.stats_deps[sname]} - self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames ) + self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames) else: if 'col_avg' in self.stats and not self.fnames: self.stats.remove('col_avg') @@ -287,8 +280,7 @@ class BlocksInfo: self.bs_keys = set( [v.key1 for v in self.fvals if v.src == 'bs'] + ['total_size'] + - (['total_weight'] if have_segwit else []) - ) + (['total_weight'] if have_segwit else [])) if 'miner' in self.fnames: # capturing parens must contain only ASCII chars! @@ -302,17 +294,16 @@ class BlocksInfo: rb'([\x20-\x7e]{9,})', rb'[/^]([a-zA-Z0-9&. #/-]{5,})', rb'[/^]([_a-zA-Z0-9&. #/-]+)/', - rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]', - )] + rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]')] - self.block_data = namedtuple('block_data',self.fnames) - self.deps = { v.src for v in self.fvals } + self.block_data = namedtuple('block_data', self.fnames) + self.deps = {v.src for v in self.fvals} - def gen_fs(self,fnames,fill=[],fill_char='-',add_name=False): + def gen_fs(self, fnames, fill=[], fill_char='-', add_name=False): for i in range(len(fnames)): name = fnames[i] - ls = (' ','')[name in self.fs_lsqueeze + self.fs_lsqueeze2] - rs = (' ','')[name in self.fs_rsqueeze] + ls = (' ', '')[name in self.fs_lsqueeze + self.fs_lsqueeze2] + rs = (' ', '')[name in self.fs_rsqueeze] if i < len(fnames) - 1 and fnames[i+1] in self.fs_lsqueeze2: rs = '' if i: @@ -321,7 +312,7 @@ class BlocksInfo: ls = '' break repl = (name if add_name else '') + ':' + (fill_char if name in fill else '') - yield (ls + self.fields[name].fs.replace(':',repl) + rs) + yield (ls + self.fields[name].fs.replace(':', repl) + rs) def conv_blkspec(self, arg): match arg: @@ -347,22 +338,22 @@ class BlocksInfo: case _: return arg - def parse_rangespec(self,arg): + def parse_rangespec(self, arg): - p = RangeParser(self,arg) + p = RangeParser(self, arg) - from_tip = p.parse('from_tip') - first,last = (self.tip-from_tip,None) if from_tip else p.parse('abs_range') - add1 = p.parse('add') - add2 = p.parse('add') + from_tip = p.parse('from_tip') + first, last = (self.tip-from_tip, None) if from_tip else p.parse('abs_range') + add1 = p.parse('add') + add2 = p.parse('add') p.finalize() if add2 and last is not None: - die(1,f'{arg!r}: invalid range specifier') + die(1, f'{arg!r}: invalid range specifier') - nblocks,step = (add1,add2) if last is None else (None,add1) + nblocks, step = (add1, add2) if last is None else (None, add1) - if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + if p.debug: msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if nblocks: if first is None: @@ -373,12 +364,12 @@ class BlocksInfo: last = self.conv_blkspec(last or first) if p.debug: - msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if first > last: - die(1,f'{first}-{last}: invalid block range') + die(1, f'{first}-{last}: invalid block range') - return self.range_data(first,last,from_tip,nblocks,step) + return self.range_data(first, last, from_tip, nblocks, step) async def process_blocks(self): @@ -388,7 +379,7 @@ class BlocksInfo: c = self.rpc - heights = self.block_list or range(self.first,self.last+1) + heights = self.block_list or range(self.first, self.last+1) self.hdrs = await get_hdrs(heights) if self.block_list: @@ -397,8 +388,7 @@ class BlocksInfo: else: self.first_prev_hdr = ( self.hdrs[0] if heights[0] == 0 else - await c.call('getblockheader',await c.call('getblockhash',heights[0]-1)) - ) + await c.call('getblockheader', await c.call('getblockhash', heights[0]-1))) self.t_cur = self.first_prev_hdr['time'] self.res = [] @@ -409,16 +399,16 @@ class BlocksInfo: ret = await self.process_block(self.hdrs[n]) self.res.append(ret) if self.fnames and not self.cfg.stats_only: - self.output_block(ret,n) + self.output_block(ret, n) - def output_block(self,data,n): + def output_block(self, data, n): def gen(): - for k,v in data._asdict().items(): + for k, v in data._asdict().items(): func = self.fields[k].fmt_func yield self.fmt_funcs[func](v) if func else v Msg(self.fs.format(*gen())) - async def process_block(self,hdr): + async def process_block(self, hdr): self.t_diff = hdr['time'] - self.t_cur self.t_cur = hdr['time'] @@ -426,14 +416,12 @@ class BlocksInfo: blk_data = { 'bh': hdr, - 'lo': { 'interval': self.t_diff } - } + 'lo': {'interval': self.t_diff}} if 'bs' in self.deps: bs = ( self.genesis_stats if hdr['height'] == 0 else - await self.rpc.call('getblockstats',hdr['hash'],list(self.bs_keys)) - ) + await self.rpc.call('getblockstats', hdr['hash'], list(self.bs_keys))) self.total_bytes += bs['total_size'] if 'total_weight' in bs: self.total_weight += bs['total_weight'] @@ -446,14 +434,13 @@ class BlocksInfo: for v in self.fvals: yield ( blk_data[v.src][v.key1] if v.key2 is None else - blk_data[v.src][v.key1][v.key2] - ) + blk_data[v.src][v.key1][v.key2]) return self.block_data(*gen()) - async def get_miner_string(self,H): - tx0 = (await self.rpc.call('getblock',H))['tx'][0] - bd = await self.rpc.call('getrawtransaction',tx0,1) + async def get_miner_string(self, H): + tx0 = (await self.rpc.call('getblock', H))['tx'][0] + bd = await self.rpc.call('getrawtransaction', tx0, 1) if type(bd) == tuple: return '---' else: @@ -464,13 +451,12 @@ class BlocksInfo: trmap_in = { '\\': ' ', '/': ' ', - ',': ' ', - } - trmap = { ord(a):b for a,b in trmap_in.items() } + ',': ' '} + trmap = {ord(a): b for a, b in trmap_in.items()} for pat in self.miner_pats: m = pat.search(cb) if m: - return re.sub( r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip() ) + return re.sub(r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip()) return '' def print_header(self): @@ -484,11 +470,11 @@ class BlocksInfo: yield self.fs.format(*hdr1) yield self.fs.format(*hdr2) - def process_stats(self,sname): - method = getattr(self,f'create_{sname}_stats',None) - return self.output_stats(method() if method else self.create_stats(sname),sname) + def process_stats(self, sname): + method = getattr(self, f'create_{sname}_stats', None) + return self.output_stats(method() if method else self.create_stats(sname), sname) - def fmt_stat_item(self,fs,s): + def fmt_stat_item(self, fs, s): return fs.format(s) if type(fs) == str else fs(s) async def output_stats(self, res, sname): @@ -497,7 +483,7 @@ class BlocksInfo: for d in data: match d: case [a, b]: - yield (indent + a).format(**{k:self.fmt_stat_item(*v) for k, v in b.items()}) + yield (indent + a).format(**{k: self.fmt_stat_item(*v) for k, v in b.items()}) case [a, _, b, c]: yield (indent + a).format(self.fmt_stat_item(b, c)) case str(): @@ -524,15 +510,14 @@ class BlocksInfo: 'range': ('{}', self.hdrs[-1]['height'] - self.hdrs[0]['height'] + 1), 'elapsed': (self.t_fmt, elapsed), 'nBlocks': ('{}', total_blks), - 'step': ('{}', self.step), - } - ) - if elapsed: - yield ( 'Start: {}', 'start_date', self.fmt_funcs['da'], self.hdrs[0]['time'] ) - yield ( 'End: {}', 'end_date', self.fmt_funcs['da'], self.hdrs[-1]['time'] ) - yield ( 'Avg BDI: {} min', 'avg_bdi', '{:.2f}', elapsed / nblocks / 60 ) + 'step': ('{}', self.step)}) - return ( 'range', gen() ) + if elapsed: + yield ('Start: {}', 'start_date', self.fmt_funcs['da'], self.hdrs[0]['time']) + yield ('End: {}', 'end_date', self.fmt_funcs['da'], self.hdrs[-1]['time']) + yield ('Avg BDI: {} min', 'avg_bdi', '{:.2f}', elapsed / nblocks / 60) + + return ('range', gen()) async def create_diff_stats(self): @@ -541,18 +526,17 @@ class BlocksInfo: tip_hdr = ( self.hdrs[-1] if self.hdrs[-1]['height'] == self.tip else - await c.call('getblockheader',await c.call('getblockhash',self.tip)) - ) + await c.call('getblockheader', await c.call('getblockhash', self.tip))) min_sample_blks = 432 # ≈3 days - rel_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-rel)) + rel_hdr = await c.call('getblockheader', await c.call('getblockhash', self.tip-rel)) if rel >= min_sample_blks: sample_blks = rel - bdi = ( tip_hdr['time'] - rel_hdr['time'] ) / rel + bdi = (tip_hdr['time'] - rel_hdr['time']) / rel else: - sample_blks = min(min_sample_blks,self.tip) - start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) + sample_blks = min(min_sample_blks, self.tip) + start_hdr = await c.call('getblockheader', await c.call('getblockhash', self.tip-sample_blks)) diff_adj = Decimal(tip_hdr['difficulty']) / Decimal(start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] @@ -560,7 +544,7 @@ class BlocksInfo: rem = self.rpc.proto.diff_adjust_interval - rel - return ( 'difficulty', ( + return ('difficulty', ( 'Difficulty Statistics:', ('Current height: {}', 'chain_tip', '{}', self.tip), ('Next diff adjust: {next_diff_adjust} (in {blks_remaining} block%s [{time_remaining}])' % suf(rem), @@ -593,105 +577,98 @@ class BlocksInfo: def gen(): for field in self.fnames: if field in self.avg_stats_skip: - yield ( field, ('{}','') ) + yield (field, ('{}', '')) else: ret = self.sum_field_avg(field) func = self.fields[field].fmt_func - yield ( field, ( (self.fmt_funcs[func] if func else '{}'), ret )) + yield (field, ((self.fmt_funcs[func] if func else '{}'), ret)) if not self.header_printed: self.print_header() - fs = ''.join(self.gen_fs(self.fnames,fill=self.avg_stats_skip,add_name=True)).strip() - return ('column_averages', ('Column averages:', (fs, dict(gen())) )) + fs = ''.join(self.gen_fs(self.fnames, fill=self.avg_stats_skip, add_name=True)).strip() + return ('column_averages', ('Column averages:', (fs, dict(gen())))) - def avg_stats_data(self,data,spec_conv,spec_val): + def avg_stats_data(self, data, spec_conv, spec_val): coin = self.rpc.proto.coin return data( hdr = 'Averages for processed blocks:', func = self.sum_field_avg, - spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}' }, + spec_sufs = {'subsidy': f' {coin}', 'totalfee': f' {coin}'}, spec_convs = { - 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), + 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'mb_per_hour': spec_conv(0, '{}'), - }, + 'mb_per_hour': spec_conv(0, '{}')}, spec_vals = ( spec_val( 'mb_per_hour', 'MB/hr', 'interval', lambda values: 'bs' in self.deps, lambda values: ( '{:.4f}'.format((self.total_bytes / 10000) / (self.total_solve_time / 36)) - if self.total_solve_time else 'N/A' ), - ), - ) - ) + if self.total_solve_time else 'N/A')), + )) mini_avg_stats_data = avg_stats_data - def total_stats_data(self,data,spec_conv,spec_val): + def total_stats_data(self, data, spec_conv, spec_val): coin = self.rpc.proto.coin return data( hdr = 'Totals for processed blocks:', func = self.sum_field_total, - spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}' }, + spec_sufs = {'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}'}, spec_convs = { - 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), + 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'reward': spec_conv(0, self.fmt_funcs['tf']), - }, + 'reward': spec_conv(0, self.fmt_funcs['tf'])}, spec_vals = ( spec_val( 'reward', 'Reward', 'totalfee', - lambda values: {'subsidy','totalfee'} <= set(values), - lambda values: values['subsidy'] + values['totalfee'] - ), - ) - ) + lambda values: {'subsidy', 'totalfee'} <= set(values), + lambda values: values['subsidy'] + values['totalfee']), + )) - async def create_stats(self,sname): + async def create_stats(self, sname): def convert_stats_hdr(field): v = self.fields[field] - return '{} {}'.format(v.hdr1.strip(), v.hdr2.strip()).replace('- ','') if v.hdr1 else v.hdr2.strip() + return '{} {}'.format( + v.hdr1.strip(), v.hdr2.strip()).replace('- ', '') if v.hdr1 else v.hdr2.strip() - d = getattr(self,f'{sname}_stats_data')( - namedtuple('stats_data',['hdr','func','spec_sufs','spec_convs','spec_vals']), - namedtuple('spec_conv',['width_adj','conv']), - namedtuple('spec_val',['name','lbl','insert_after','condition','code']) - ) + d = getattr(self, f'{sname}_stats_data')( + namedtuple('stats_data', ['hdr', 'func', 'spec_sufs', 'spec_convs', 'spec_vals']), + namedtuple('spec_conv', ['width_adj', 'conv']), + namedtuple('spec_val', ['name', 'lbl', 'insert_after', 'condition', 'code'])) fnames = [n for n in self.fnames if n in self.stats_deps[sname]] - lbls = {n:convert_stats_hdr(n) for n in fnames} - values = {n:d.func(n) for n in fnames} - col1_w = max((len(l) for l in lbls.values()),default=0) + 2 + lbls = {n: convert_stats_hdr(n) for n in fnames} + values = {n: d.func(n) for n in fnames} + col1_w = max((len(l) for l in lbls.values()), default=0) + 2 + print(d.spec_vals) for v in d.spec_vals: + print(v) if v.condition(values): try: idx = fnames.index(v.insert_after) + 1 except: idx = 0 - fnames.insert(idx,v.name) + fnames.insert(idx, v.name) lbls[v.name] = v.lbl values[v.name] = v.code(values) def gen(): - for n,fname in enumerate(fnames): + for n, fname in enumerate(fnames): spec_conv = d.spec_convs.get(fname) yield ( '{lbl:{wid}} {{}}{suf}'.format( lbl = lbls[fname] + ':', wid = col1_w + (spec_conv.width_adj if spec_conv else 0), - suf = d.spec_sufs.get(fname) or '' - ), + suf = d.spec_sufs.get(fname) or ''), fname, spec_conv.conv if spec_conv else ( - (lambda x: self.fmt_funcs[x] if x else '{}')(self.fields[fname].fmt_func) - ), - values[fname] - ) + (lambda x: self.fmt_funcs[x] if x else '{}')(self.fields[fname].fmt_func)), + values[fname]) - return ( sname, (d.hdr,) + tuple(gen()) ) + return (sname, (d.hdr,) + tuple(gen())) - def process_stats_pre(self,i): + def process_stats_pre(self, i): if (self.fnames and not self.cfg.stats_only) or i != 0: Msg('') @@ -724,13 +701,12 @@ class BlocksInfo: 'totalfee': 0, 'txs': 1, 'utxo_increase': 1, - 'utxo_size_inc': 117 - } + 'utxo_size_inc': 117} class JSONBlocksInfo(BlocksInfo): - def __init__(self,cfg,cmd_args,rpc): - super().__init__(cfg,cmd_args,rpc) + def __init__(self, cfg, cmd_args, rpc): + super().__init__(cfg, cmd_args, rpc) if self.cfg.json_raw: self.output_block = self.output_block_raw self.fmt_stat_item = self.fmt_stat_item_raw @@ -741,22 +717,22 @@ class JSONBlocksInfo(BlocksInfo): await super().process_blocks() Msg_r(']') - def output_block_raw(self,data,n): - Msg_r( (', ','')[n==0] + json.dumps(data._asdict(),cls=json_encoder) ) + def output_block_raw(self, data, n): + Msg_r((', ', '')[n==0] + json.dumps(data._asdict(), cls=json_encoder)) - def output_block(self,data,n): + def output_block(self, data, n): def gen(): - for k,v in data._asdict().items(): + for k, v in data._asdict().items(): func = self.fields[k].fmt_func - yield ( k, (self.fmt_funcs[func](v) if func else v) ) - Msg_r( (', ','')[n==0] + json.dumps(dict(gen()),cls=json_encoder) ) + yield (k, (self.fmt_funcs[func](v) if func else v)) + Msg_r((', ', '')[n==0] + json.dumps(dict(gen()), cls=json_encoder)) def print_header(self): pass - def fmt_stat_item_raw(self,fs,s): + def fmt_stat_item_raw(self, fs, s): return s - async def output_stats(self,res,sname): + async def output_stats(self, res, sname): def gen(data): for d in data: @@ -771,10 +747,10 @@ class JSONBlocksInfo(BlocksInfo): case _: assert False, f'{d}: invalid stats data' - varname,data = await res - Msg_r(', "{}_data": {}'.format( varname, json.dumps(dict(gen(data)),cls=json_encoder) )) + varname, data = await res + Msg_r(', "{}_data": {}'.format(varname, json.dumps(dict(gen(data)), cls=json_encoder))) - def process_stats_pre(self,i): pass + def process_stats_pre(self, i): pass def finalize_output(self): Msg('}') diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index c3d7c68..3329f07 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -14,66 +14,66 @@ mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes import asyncio from collections import namedtuple -from mmgen.util import msg,msg_r,is_int -from mmgen.term import get_term,get_terminal_size,get_char +from mmgen.util import msg, msg_r, is_int +from mmgen.term import get_term, get_terminal_size, get_char from mmgen.ui import line_input from .PollDisplay import PollDisplay -RED,RESET = ('\033[31m','\033[0m') -COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247,256)) + [231]] -ERASE_ALL,CUR_HOME = ('\033[J','\033[H') -CUR_HIDE,CUR_SHOW = ('\033[?25l','\033[?25h') +RED, RESET = ('\033[31m', '\033[0m') +COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247, 256)) + [231]] +ERASE_ALL, CUR_HOME = ('\033[J', '\033[H') +CUR_HIDE, CUR_SHOW = ('\033[?25l', '\033[?25h') term = None class Display(PollDisplay): poll_secs = 2 - def __init__(self,cfg): + def __init__(self, cfg): super().__init__(cfg) - global term,term_width + global term, term_width if not term: term = get_term() term.init(noecho=True) term_width = self.cfg.columns or get_terminal_size().width msg_r(CUR_HOME+ERASE_ALL+CUR_HOME) - async def get_info(self,rpc): + async def get_info(self, rpc): return await rpc.call('getpeerinfo') - def display(self,count): + def display(self, count): msg_r( CUR_HOME + (ERASE_ALL if count == 1 else '') + 'CONNECTED PEERS ({a}) {b} - poll {c}'.format( a = len(self.info), b = self.desc, - c = count ).ljust(term_width)[:term_width] + c = count).ljust(term_width)[:term_width] + '\n' + ('\n'.join(self.gen_display()) + '\n' if self.info else '') + ERASE_ALL + f"Type a peer number to disconnect, 'q' to quit, or any other key for {self.other_desc} display:" - + '\b' ) + + '\b') - async def disconnect_node(self,rpc,addr): - return await rpc.call('disconnectnode',addr) + async def disconnect_node(self, rpc, addr): + return await rpc.call('disconnectnode', addr) def get_input(self): - s = get_char(immed_chars='q0123456789',prehold_protect=False,num_bytes=1) + s = get_char(immed_chars='q0123456789', prehold_protect=False, num_bytes=1) if not is_int(s): return s with self.info_lock: msg('') term.reset() # readline required for correct operation here; without it, user must re-type first digit - ret = line_input( self.cfg, 'peer number> ', insert_txt=s, hold_protect=False ) + ret = line_input(self.cfg, 'peer number> ', insert_txt=s, hold_protect=False) term.init(noecho=True) self.enable_display = False # prevent display from updating before process_input() return ret - async def process_input(self,rpc): + async def process_input(self, rpc): ids = tuple(str(i['id']) for i in self.info) ret = False @@ -83,7 +83,7 @@ class Display(PollDisplay): from mmgen.exception import RPCFailure addr = self.info[ids.index(self.input)]['addr'] try: - await self.disconnect_node(rpc,addr) + await self.disconnect_node(rpc, addr) except RPCFailure: msg_r(f'Unable to disconnect peer {self.input} ({addr})') else: @@ -105,8 +105,8 @@ class BlocksDisplay(Display): def gen_display(self): - pd = namedtuple('peer_data',['id','blks_data','blks_width']) - bd = namedtuple('block_datum',['num','disp']) + pd = namedtuple('peer_data', ['id', 'blks_data', 'blks_width']) + bd = namedtuple('block_datum', ['num', 'disp']) def gen_block_data(): global min_height @@ -114,15 +114,15 @@ class BlocksDisplay(Display): for d in self.info: if d.get('inflight'): blocks = d['inflight'] - min_height = min(blocks) if not min_height else min(min_height,min(blocks)) - line = ' '.join(map(str,blocks))[:blks_field_width] + min_height = min(blocks) if not min_height else min(min_height, min(blocks)) + line = ' '.join(map(str, blocks))[:blks_field_width] blocks_disp = line.split() yield pd( d['id'], - [bd(blocks[i],blocks_disp[i]) for i in range(len(blocks_disp))], - len(line) ) + [bd(blocks[i], blocks_disp[i]) for i in range(len(blocks_disp))], + len(line)) else: - yield pd(d['id'],[],0) + yield pd(d['id'], [], 0) def gen_line(peer_data): for blk in peer_data.blks_data: @@ -136,7 +136,7 @@ class BlocksDisplay(Display): for peer_data in tuple(gen_block_data()): yield fs.format( peer_data.id, - ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width) ) + ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width)) class PeersDisplay(Display): @@ -152,5 +152,4 @@ class PeersDisplay(Display): A = id_width, b = d['addr'], B = addr_width, - c = d['subver'] - ).ljust(term_width)[:term_width] + c = d['subver']).ljust(term_width)[:term_width] diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 1de8a1a..8274e40 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -12,7 +12,7 @@ mmgen_node_tools.PollDisplay: update and display RPC data; get input from user """ -import sys,threading +import sys, threading from mmgen.util import msg from mmgen.term import get_char @@ -22,18 +22,18 @@ class PollDisplay: input = None poll_secs = 1 - def __init__(self,cfg): + def __init__(self, cfg): self.cfg = cfg self.info_lock = threading.Lock() # self.info accessed by 2 threads self.display_kill_flag = threading.Event() def get_input(self): - return get_char(immed_chars='q',prehold_protect=False,num_bytes=1) + return get_char(immed_chars='q', prehold_protect=False, num_bytes=1) - async def process_input(self,rpc): + async def process_input(self, rpc): return True - async def run(self,rpc): + async def run(self, rpc): async def do_display(): with self.info_lock: @@ -68,7 +68,7 @@ class PollDisplay: self.display_kill_flag.set() while True: - threading.Thread(target=get_input,daemon=True).start() + threading.Thread(target=get_input, daemon=True).start() await do_display() if await process_input(): break diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index df9f8ae..557b97a 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -19,28 +19,28 @@ mmgen_node_tools.Sound: audio-related functions for MMGen node tools """ -import sys,os,time +import sys, os, time from mmgen.util import die from mmgen_node_tools.Util import do_system _alsa_config_file = '/tmp/alsa-config-' + os.path.basename(sys.argv[0]) -_dvols = { 'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190 } +_dvols = {'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190} def timespec2secs(ts): import re - mul = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24 } + mul = {'s': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24} pat = r'^([0-9]+)([smhd]*)$' - m = re.match(pat,ts) + m = re.match(pat, ts) if m is None: die(2,"'%s': invalid time specifier" % ts) - a,b = m.groups() + a, b = m.groups() return int(a) * (mul[b] if b else 1) def parse_repeat_spec(rs): - return [(timespec2secs(i),timespec2secs(j)) - for i,j in [a.split(':') for a in rs.split(',')]] + return [(timespec2secs(i), timespec2secs(j)) + for i, j in [a.split(':') for a in rs.split(',')]] def init_sound(): def _restore_sound(): @@ -51,33 +51,33 @@ def init_sound(): atexit.register(_restore_sound) do_system('sudo alsactl store -f ' + _alsa_config_file) -def play_sound(fn,vol,repeat_spec='',remote_host='',kill_flg=None,testing=False): +def play_sound(fn, vol, repeat_spec='', remote_host='', kill_flg=None, testing=False): if not remote_host: do_system('sudo alsactl store -f ' + _alsa_config_file) - for k in 'Master','Speaker','Headphone': - do_system(('sudo amixer -q set %s on' % k),testing) + for k in 'Master', 'Speaker', 'Headphone': + do_system(('sudo amixer -q set %s on' % k), testing) # do_system('amixer -q set Headphone off') - vols = dict([(k,int(_dvols[k] * float(vol) / 100)) for k in _dvols]) + vols = dict([(k, int(_dvols[k] * float(vol) / 100)) for k in _dvols]) for k in vols: - do_system('sudo amixer -q set %s %s' % (k,vols[k]),testing) + do_system('sudo amixer -q set %s %s' % (k, vols[k]), testing) fn = os.path.expanduser(fn) cmd = ( 'aplay -q %s' % fn, - 'ssh %s mmnode-play-sound -v%d %s' % (remote_host,vol,fn) + 'ssh %s mmnode-play-sound -v%d %s' % (remote_host, vol, fn) )[bool(remote_host)] if repeat_spec and kill_flg: - for interval,duration in parse_repeat_spec(repeat_spec): + for interval, duration in parse_repeat_spec(repeat_spec): start = time.time() while time.time() < start + duration: - do_system(cmd,testing) + do_system(cmd, testing) if kill_flg.wait(interval): if not remote_host: do_system('sudo alsactl restore -f ' + _alsa_config_file) return else: # Play once - do_system(cmd,testing) + do_system(cmd, testing) if not remote_host: do_system('sudo alsactl restore -f ' + _alsa_config_file) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 95e03f2..ebefcf3 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -29,10 +29,10 @@ from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, lis from mmgen.ui import do_pager homedir = os.getenv('HOME') -dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') +dfl_cachedir = os.path.join(homedir, '.cache', 'mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' -asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) +asset_tuple = namedtuple('asset_tuple', ['symbol', 'id', 'source']) last_api_host = None percent_cols = { @@ -50,20 +50,19 @@ class DataSource: }, { 'fi': 'yahoospot', 'hi': 'yahoohist', - } - ] + }] @classmethod - def get_sources(cls,randomize=False): - g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups - return {k:v for a in g for k,v in a.items()} + def get_sources(cls, randomize=False): + g = random.sample(cls.source_groups, k=len(cls.source_groups)) if randomize else cls.source_groups + return {k: v for a in g for k, v in a.items()} class base: def fetch_delay(self): global last_api_host if not gcfg.testing and last_api_host and last_api_host != self.api_host: - delay = 1 + random.randrange(1,5000) / 1000 + delay = 1 + random.randrange(1, 5000) / 1000 msg_r(f'Waiting {delay:.3f} seconds...') time.sleep(delay) msg('') @@ -72,19 +71,18 @@ class DataSource: def get_data_from_network(self): curl_cmd = list_gen( - ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], + ['curl', '--tr-encoding', '--header', 'Accept: application/json', True], ['--compressed'], # adds 'Accept-Encoding: gzip' - ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], + ['--proxy', cfg.proxy, isinstance(cfg.proxy, str)], ['--silent', not cfg.verbose], - [self.api_url] - ) + [self.api_url]) if gcfg.testing: - Msg(fmt_list(curl_cmd,fmt='bare')) + Msg(fmt_list(curl_cmd, fmt='bare')) return try: - return run(curl_cmd,check=True,stdout=PIPE).stdout.decode() + return run(curl_cmd, check=True, stdout=PIPE).stdout.decode() except CalledProcessError as e: msg('') from .Misc import curl_exit_codes @@ -101,7 +99,7 @@ class DataSource: os.makedirs(cfg.cachedir) if not os.path.exists(self.json_fn): - open(self.json_fn,'w').write('{}') + open(self.json_fn, 'w').write('{}') use_cached_data = cfg.cached_data and not gcfg.download @@ -123,7 +121,7 @@ class DataSource: if gcfg.testing: return {} else: - die(1,self.rate_limit_errmsg(elapsed)) + die(1, self.rate_limit_errmsg(elapsed)) match data_type: case 'json': @@ -131,7 +129,7 @@ class DataSource: data = json.loads(data_in) except: self.json_data_error_msg(data_in) - die(2,'Retrieved data is not valid JSON, exiting') + die(2, 'Retrieved data is not valid JSON, exiting') json_text = data_in case 'python': data = data_in @@ -143,9 +141,9 @@ class DataSource: f'No cached {self.data_desc}! Run command without the --cached-data option, ' 'or use --download to retrieve data from remote host') else: - die(2,'Remote host returned no data!') + die(2, 'Remote host returned no data!') elif 'error' in data: - die(1,data['error']) + die(1, data['error']) if use_cached_data: if not cfg.quiet: @@ -162,15 +160,15 @@ class DataSource: return self.postprocess_data(data) - def json_data_error_msg(self,json_text): + def json_data_error_msg(self, json_text): pass - def postprocess_data(self,data): + def postprocess_data(self, data): return data @property def json_fn_rel(self): - return os.path.relpath(self.json_fn,start=homedir) + return os.path.relpath(self.json_fn, start=homedir) class coinpaprika(base): desc = 'CoinPaprika' @@ -185,30 +183,29 @@ class DataSource: def __init__(self): self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) - def rate_limit_errmsg(self,elapsed): + def rate_limit_errmsg(self, elapsed): return ( f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + - ('' if cfg.btc_only else ', or use --cached-data or --btc') - ) + ('' if cfg.btc_only else ', or use --cached-data or --btc')) @property def api_url(self): return ( f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else - f'https://{self.api_host}/v1/tickers' ) + f'https://{self.api_host}/v1/tickers') @property def json_fn(self): return os.path.join( cfg.cachedir, - 'ticker-btc.json' if cfg.btc_only else 'ticker.json' ) + 'ticker-btc.json' if cfg.btc_only else 'ticker.json') @property def timeout(self): return 0 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit - def json_data_error_msg(self,json_text): + def json_data_error_msg(self, json_text): tor_captcha_msg = f""" If you’re using Tor, the API request may have failed due to Captcha protection. A workaround for this issue is to retrieve the JSON data with a browser from @@ -223,20 +220,20 @@ class DataSource: Then invoke the program with --cached-data and without --btc """ msg(json_text[:1024] + '...') - msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) + msg(orange(fmt(tor_captcha_msg, strip_char='\t'))) - def postprocess_data(self,data): + def postprocess_data(self, data): return [data] if cfg.btc_only else data @staticmethod - def parse_asset_id(s,require_label): - sym,label = (*s.split('-',1),None)[:2] + def parse_asset_id(s, require_label): + sym, label = (*s.split('-', 1), None)[:2] if require_label and not label: - die(1,f'{s!r}: asset label is missing') + die(1, f'{s!r}: asset label is missing') return asset_tuple( symbol = sym.upper(), id = (s.lower() if label else None), - source = 'cc' ) + source = 'cc') class yahoospot(base): @@ -250,12 +247,12 @@ class DataSource: json_fn_basename = 'ticker-finance.json' @staticmethod - def get_id(sym,data): + def get_id(sym, data): return sym.lower() @staticmethod - def conv_data(sym,data,btcusd): - price_usd = Decimal( data['regularMarketPrice']['raw'] ) + def conv_data(sym, data, btcusd): + price_usd = Decimal(data['regularMarketPrice']['raw']) return { 'id': sym, 'name': data['shortName'], @@ -266,15 +263,14 @@ class DataSource: 'percent_change_30d': data['pct_chg_4wks'], 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, - 'last_updated': data['regularMarketTime'], - } + 'last_updated': data['regularMarketTime']} - def rate_limit_errmsg(self,elapsed): + def rate_limit_errmsg(self, elapsed): return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' @property def json_fn(self): - return os.path.join( cfg.cachedir, self.json_fn_basename ) + return os.path.join(cfg.cachedir, self.json_fn_basename) @property def timeout(self): @@ -282,37 +278,36 @@ class DataSource: @property def symbols(self): - return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + return [r.symbol for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi'] def get_data_from_network(self): kwargs = { 'formatted': True, 'asynchronous': True, - 'proxies': { 'https': cfg.proxy2 }, - } + 'proxies': {'https': cfg.proxy2}} if gcfg.test_suite: - kwargs.update({ 'timeout': 1, 'retry': 0 }) + kwargs.update({'timeout': 1, 'retry': 0}) if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( self.symbols, - fmt_dict(kwargs,fmt='kwargs') )) + fmt_dict(kwargs, fmt='kwargs'))) return from yahooquery import Ticker - return self.process_network_data( Ticker(self.symbols,**kwargs) ) + return self.process_network_data(Ticker(self.symbols,**kwargs)) - def process_network_data(self,ticker): + def process_network_data(self, ticker): return ticker.price @staticmethod - def parse_asset_id(s,require_label): + def parse_asset_id(s, require_label): return asset_tuple( symbol = s.upper(), id = s.lower(), - source = 'fi' ) + source = 'fi') class yahoohist(yahoospot): @@ -322,17 +317,17 @@ class DataSource: period = '1y' interval = '1wk' - def process_network_data(self,ticker): + def process_network_data(self, ticker): return ticker.history( period = self.period, interval = self.interval).to_json(orient='index') - def postprocess_data(self,data): + def postprocess_data(self, data): def gen(): keys = set() d = {} for key, val in data.items(): - if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): + if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$", key): date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) if (sym := m[1]) in keys: d[date] = val @@ -343,12 +338,12 @@ class DataSource: return dict(gen()) def assets_list_gen(cfg_in): - for k,v in cfg_in.cfg['assets'].items(): + for k, v in cfg_in.cfg['assets'].items(): yield '' yield k.upper() for e in v: - out = e.split('-',1) - yield ' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '') + out = e.split('-', 1) + yield ' {:5s} {}'.format(out[0], out[1] if len(out) == 2 else '') def gen_data(data): """ @@ -367,10 +362,9 @@ def gen_data(data): f'The symbol {dup_sym!r} is shared by the following assets:\n' + '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + - f'instead of {dup_sym!r}' - ) + f'instead of {dup_sym!r}') - def check_assets_found(wants,found,keys=['symbol','id']): + def check_assets_found(wants, found, keys=['symbol', 'id']): error = False for k in keys: missing = wants[k] - found[k] @@ -378,36 +372,32 @@ def gen_data(data): msg( ('The following IDs were not found in source data:\n{}' if k == 'id' else 'The following symbols could not be resolved:\n{}').format( - fmt_list(missing,fmt='col',indent=' ') - )) + fmt_list(missing, fmt='col', indent=' '))) error = True if error: - die(1,'Missing data, exiting') + die(1, 'Missing data, exiting') rows_want = { - 'id': {r.id for r in cfg.rows if isinstance(r,tuple) and r.id} - {'usd-us-dollar'}, - 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, - } + 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) usr_rate_assets_want = { 'id': {a.id for a in usr_rate_assets if a.id}, - 'symbol': {a.symbol for a in usr_rate_assets if not a.id} - } + 'symbol': {a.symbol for a in usr_rate_assets if not a.id}} usr_assets = cfg.usr_rows + cfg.usr_columns + tuple(c for c in (cfg.query or ()) if c) usr_wants = { 'id': ( {a.id for a in usr_assets + usr_rate_assets if a.id} - - {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'} ) + {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'}) , 'symbol': ( {a.symbol for a in usr_assets + usr_rate_assets if not a.id} - - {a.symbol for a in usr_assets if a.rate} - {'USD'} ), - } + {a.symbol for a in usr_assets if a.rate} - {'USD'})} - found = { 'id': set(), 'symbol': set() } + found = {'id': set(), 'symbol': set()} rate_assets = {} - wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} + wants = {k: rows_want[k] | usr_wants[k] for k in ('id', 'symbol')} for d in data['cc']: if d['id'] == 'btc-bitcoin': @@ -417,38 +407,37 @@ def gen_data(data): get_id = src_cls['fi'].get_id conv_func = src_cls['fi'].conv_data - for k,v in data['fi'].items(): - id = get_id(k,v) + for k, v in data['fi'].items(): + id = get_id(k, v) if wants['id']: if id in wants['id']: - if not isinstance(v,dict): + if not isinstance(v, dict): die(2, str(v)) if id in found['id']: - die(1,dup_sym_errmsg(id)) + die(1, dup_sym_errmsg(id)) if m := data['hi'].get(k): spot = v['regularMarketPrice']['raw'] hist = tuple(m.values()) v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( (spot / hist[-2]['close'] - 1) * 100, (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month - (spot / hist[0]['close'] - 1) * 100, - ) + (spot / hist[0]['close'] - 1) * 100) else: v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None - yield ( id, conv_func(id,v,btcusd) ) + yield (id, conv_func(id, v, btcusd)) found['id'].add(id) wants['id'].remove(id) if id in usr_rate_assets_want['id']: - rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key + rate_assets[k] = conv_func(id, v, btcusd) # NB: using symbol instead of ID for key else: break - for k in ('id','symbol'): + for k in ('id', 'symbol'): for d in data['cc']: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: - die(1,dup_sym_errmsg(d[k])) + die(1, dup_sym_errmsg(d[k])) if not 'price_usd' in d: d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd @@ -457,8 +446,9 @@ def gen_data(data): d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] # .replace('Z','+00:00') -- Python 3.9 backport - d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated'].replace('Z','+00:00')).timestamp()) - yield (d['id'],d) + d['last_updated'] = int(datetime.datetime.fromisoformat( + d['last_updated'].replace('Z', '+00:00')).timestamp()) + yield (d['id'], d) found[k].add(d[k]) wants[k].remove(d[k]) if d[k] in usr_rate_assets_want[k]: @@ -466,7 +456,7 @@ def gen_data(data): else: break - check_assets_found(usr_wants,found) + check_assets_found(usr_wants, found) for asset in (cfg.usr_rows + cfg.usr_columns): if asset.rate: @@ -475,14 +465,13 @@ def gen_data(data): """ _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() ra_rate = rate_assets[asset.rate_asset.symbol]['price_usd'] if asset.rate_asset else 1 - yield ( _id, { + yield (_id, { 'symbol': asset.symbol, 'id': _id, 'name': ' '.join(_id.split('-')[1:]), 'price_usd': ra_rate / asset.rate, 'price_btc': ra_rate / asset.rate / btcusd, - 'last_updated': None, - }) + 'last_updated': None}) yield ('usd-us-dollar', { 'symbol': 'USD', @@ -490,21 +479,20 @@ def gen_data(data): 'name': 'US Dollar', 'price_usd': Decimal(1), 'price_btc': Decimal(1) / btcusd, - 'last_updated': None, - }) + 'last_updated': None}) def main(): def update_sample_file(usr_cfg_file): - usr_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() + usr_data = files('mmgen_node_tools').joinpath('data', os.path.basename(usr_cfg_file)).read_text() sample_file = usr_cfg_file + '.sample' sample_data = open(sample_file).read() if os.path.exists(sample_file) else None if usr_data != sample_data: - os.makedirs(os.path.dirname(sample_file),exist_ok=True) + os.makedirs(os.path.dirname(sample_file), exist_ok=True) msg('{} {}'.format( - ('Updating','Creating')[sample_data is None], - sample_file )) - open(sample_file,'w').write(usr_data) + ('Updating', 'Creating')[sample_data is None], + sample_file)) + open(sample_file, 'w').write(usr_data) try: from importlib.resources import files # Python 3.9 @@ -515,19 +503,19 @@ def main(): update_sample_file(cfg_in.portfolio_file) if gcfg.portfolio and not cfg_in.portfolio: - die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( - os.path.relpath(cfg_in.portfolio_file,start=homedir))) + die(1, 'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( + os.path.relpath(cfg_in.portfolio_file, start=homedir))) if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: if not gcfg.download in DataSource.get_sources(): - die(1,f'{gcfg.download!r}: invalid data source') + die(1, f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: src_ids = DataSource.get_sources(randomize=True) - src_data = { k: src_cls[k]().get_data() for k in src_ids } + src_data = {k: src_cls[k]().get_data() for k in src_ids} if gcfg.testing: return @@ -542,24 +530,24 @@ def main(): data = dict(gen_data(src_data)) (do_pager if cfg.pager else Msg_r)( - '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n') + '\n'.join(getattr(Ticker, cfg.clsname)(data).gen_output()) + '\n') def make_cfg(gcfg_arg): - query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) + query_tuple = namedtuple('query', ['asset', 'to_asset']) + asset_data = namedtuple('asset_data', ['symbol', 'id', 'amount', 'rate', 'rate_asset', 'source']) - def parse_asset_id(s,require_label=False): - return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) + def parse_asset_id(s, require_label=False): + return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) def get_rows_from_cfg(add_data=None): def gen(): - for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): + for n, (k, v) in enumerate(cfg_in.cfg['assets'].items()): yield k if add_data and k in add_data: v += tuple(add_data[k]) for e in v: - yield parse_asset_id(e,require_label=True) + yield parse_asset_id(e, require_label=True) return tuple(gen()) def parse_percent_cols(arg): @@ -568,17 +556,19 @@ def make_cfg(gcfg_arg): res = arg.lower().split(',') for s in res: if s not in percent_cols: - die(1,f'{arg!r}: invalid --percent-cols parameter (valid letters: {fmt_list(percent_cols)})') + die(1, '{!r}: invalid --percent-cols parameter (valid letters: {})'.format( + arg, + fmt_list(percent_cols))) return res - def parse_usr_asset_arg(key,use_cf_file=False): + def parse_usr_asset_arg(key, use_cf_file=False): """ asset_id[:rate[:rate_asset]] """ def parse_parm(s): ss = s.split(':') - assert len(ss) in (1,2,3), f'{s}: malformed argument' - asset_id,rate,rate_asset = (*ss,None,None)[:3] + assert len(ss) in (1, 2, 3), f'{s}: malformed argument' + asset_id, rate, rate_asset = (*ss, None, None)[:3] parsed_id = parse_asset_id(asset_id) return asset_data( @@ -588,19 +578,19 @@ def make_cfg(gcfg_arg): rate = ( None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else - Decimal(rate) ), + Decimal(rate)), rate_asset = parse_asset_id(rate_asset) if rate_asset else None, - source = parsed_id.source ) + source = parsed_id.source) - cl_opt = getattr(gcfg,key) + cl_opt = getattr(gcfg, key) cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] - return tuple( parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt) ) + return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) def parse_query_arg(s): """ asset_id:amount[:to_asset_id[:to_amount]] """ - def parse_query_asset(asset_id,amount): + def parse_query_asset(asset_id, amount): parsed_id = parse_asset_id(asset_id) return asset_data( symbol = parsed_id.symbol, @@ -608,21 +598,20 @@ def make_cfg(gcfg_arg): amount = None if amount is None else Decimal(amount), rate = None, rate_asset = None, - source = parsed_id.source ) + source = parsed_id.source) ss = s.split(':') - assert len(ss) in (2,3,4), f'{s}: malformed argument' - asset_id,amount,to_asset_id,to_amount = (*ss,None,None)[:4] + assert len(ss) in (2, 3, 4), f'{s}: malformed argument' + asset_id, amount, to_asset_id, to_amount = (*ss, None, None)[:4] return query_tuple( - asset = parse_query_asset(asset_id,amount), - to_asset = parse_query_asset(to_asset_id,to_amount) if to_asset_id else None - ) + asset = parse_query_asset(asset_id, amount), + to_asset = parse_query_asset(to_asset_id, to_amount) if to_asset_id else None) - def gen_uniq(obj_list,key,preload=None): - found = set([getattr(obj,key) for obj in preload if hasattr(obj,key)] if preload else ()) + def gen_uniq(obj_list, key, preload=None): + found = set([getattr(obj, key) for obj in preload if hasattr(obj, key)] if preload else ()) for obj in obj_list: - id = getattr(obj,key) + id = getattr(obj, key) if id not in found: yield obj found.add(id) @@ -632,39 +621,38 @@ def make_cfg(gcfg_arg): 'user_added', usr_rows + (tuple(asset for asset in query if asset) if query else ()) + - usr_columns ) + usr_columns) def get_portfolio_assets(ret=()): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e,require_label=True) for e in cfg_in.portfolio) - return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') ) + ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) + return ('portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC')) def get_portfolio(): - return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'} + return {k: Decimal(v) for k, v in cfg_in.portfolio.items() + if (not gcfg.btc) or k == 'btc-bitcoin'} def parse_add_precision(arg): if not arg: return 0 s = str(arg) if not (s.isdigit() and s.isascii()): - die(1,f'{s}: invalid parameter for --add-precision (not an integer)') + die(1, f'{s}: invalid parameter for --add-precision (not an integer)') if int(s) > 30: - die(1,f'{s}: invalid parameter for --add-precision (value >30)') + die(1, f'{s}: invalid parameter for --add-precision (value >30)') return int(s) def create_rows(): rows = ( ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin',parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None ) - ) + ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else + get_rows_from_cfg(add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None)) - for hdr,data in ( - (get_usr_assets(),) if query else - (get_usr_assets(), get_portfolio_assets()) - ): + for hdr, data in ( + (get_usr_assets(),) if query else + (get_usr_assets(), get_portfolio_assets())): if data: - uniq_data = tuple(gen_uniq(data,'symbol',preload=rows)) + uniq_data = tuple(gen_uniq(data, 'symbol', preload=rows)) if uniq_data: rows += (hdr,) + uniq_data return rows @@ -693,26 +681,25 @@ def make_cfg(gcfg_arg): 'quiet', 'verbose']) - global gcfg,cfg_in,src_cls,cfg + global gcfg, cfg_in, src_cls, cfg gcfg = gcfg_arg - src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() } + src_cls = {k: getattr(DataSource, v) for k, v in DataSource.get_sources().items()} fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args cfg_in = get_cfg_in() usr_rows = parse_usr_asset_arg('add_rows') - usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) + usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None def get_proxy(name): - proxy = getattr(gcfg,name) + proxy = getattr(gcfg, name) return ( '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') - else (proxy or cfg_in.cfg.get(name)) - ) + else (proxy or cfg_in.cfg.get(name))) proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy @@ -723,7 +710,7 @@ def make_cfg(gcfg_arg): usr_rows = usr_rows, usr_columns = usr_columns, query = query, - adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ), + adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', btc_only = gcfg.btc or cfg_in.cfg.get('btc'), add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), @@ -745,17 +732,16 @@ def make_cfg(gcfg_arg): thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), - verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), - ) + verbose = gcfg.verbose or cfg_in.cfg.get('verbose')) def get_cfg_in(): - ret = namedtuple('cfg_in_data',['cfg','portfolio','cfg_file','portfolio_file']) - cfg_file,portfolio_file = ( - [os.path.join(gcfg.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)] - ) - cfg_data,portfolio_data = ( - [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None for fn in (cfg_file,portfolio_file)] - ) + ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) + cfg_file, portfolio_file = ( + [os.path.join(gcfg.data_dir_root, 'node_tools', fn) + for fn in (cfg_fn, portfolio_fn)]) + cfg_data, portfolio_data = ( + [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None + for fn in (cfg_file, portfolio_file)]) return ret( cfg = cfg_data or { 'assets': { @@ -765,14 +751,11 @@ def get_cfg_in(): # Pound Sterling, Euro, Swiss Franc 'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ], # Dow Jones Industrials, Nasdaq 100, S&P 500 - 'index': [ '^dji', '^ixic', '^gspc' ], - }, - 'proxy': 'http://vpn-gw:8118' - }, - portfolio = portfolio_data, - cfg_file = cfg_file, - portfolio_file = portfolio_file, - ) + 'index': [ '^dji', '^ixic', '^gspc' ]}, + 'proxy': 'http://vpn-gw:8118'}, + portfolio = portfolio_data, + cfg_file = cfg_file, + portfolio_file = portfolio_file) class Ticker: @@ -781,43 +764,43 @@ class Ticker: offer = None to_asset = None - def __init__(self,data): + def __init__(self, data): self.comma = ',' if cfg.thousands_comma else '' - self.col1_wid = max(len('TOTAL'),( + self.col1_wid = max(len('TOTAL'), ( max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else - max(len(d['symbol']) for d in data.values()) - )) + 1 + max(len(d['symbol']) for d in data.values()))) + 1 - self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows] - self.col_usd_prices = {k:self.data[k]['price_usd'] for k in self.col_ids} + self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row + for row in cfg.rows] + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} - self.prices = {row.id:self.get_row_prices(row.id) - for row in self.rows if isinstance(row,tuple) and row.id in data} + self.prices = {row.id: self.get_row_prices(row.id) + for row in self.rows if isinstance(row, tuple) and row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') - def format_last_update_col(self,cross_assets=()): + def format_last_update_col(self, cross_assets=()): if cfg.elapsed: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X', time.gmtime(t)) + fmt_func = lambda t, now: time.strftime('%F %X', time.gmtime(t)) d = self.data max_w = 0 if cross_assets: last_updated_x = [d[a.id]['last_updated'] for a in cross_assets] - min_t = min( (int(n) for n in last_updated_x if isinstance(n,int) ), default=None ) + min_t = min((int(n) for n in last_updated_x if isinstance(n, int)), default=None) else: min_t = None for row in self.rows: - if isinstance(row,tuple): + if isinstance(row, tuple): try: - t = int( d[row.id]['last_updated'] ) + t = int(d[row.id]['last_updated']) except TypeError as e: d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) except KeyError as e: @@ -825,7 +808,7 @@ class Ticker: pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( - (min(t,min_t) if min_t else t), + (min(t, min_t) if min_t else t), now = now) max_w = max(len(t_fmt), max_w) @@ -836,7 +819,7 @@ class Ticker: self.uprec = {k: max(0, v+4) + cfg.add_prec for k, v in exp} self.uwid = {k: 12 + max(0, abs(v)-6) + cfg.add_prec for k, v in exp} - def get_id(self,asset): + def get_id(self, asset): if asset.id: return asset.id else: @@ -844,11 +827,11 @@ class Ticker: if d['symbol'] == asset.symbol: return d['id'] - def create_label(self,id): + def create_label(self, id): return self.data[id]['name'].upper() def gen_output(self): - yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now))) + yield 'Current time: {} UTC'.format(time.strftime('%F %X', time.gmtime(now))) for asset in self.usr_col_assets: if asset.symbol != 'USD': @@ -858,17 +841,16 @@ class Ticker: self.create_label(asset.id), usdprice, self.comma, - max(2, 4-usdprice.adjusted()) ) + max(2, 4-usdprice.adjusted())) - if hasattr(self,'subhdr'): + if hasattr(self, 'subhdr'): yield self.subhdr if self.show_adj: yield ( ('Offered price differs from spot' if self.offer else 'Adjusting prices') + ' by ' - + yellow('{:+.2f}%'.format( (self.adjust-1) * 100 )) - ) + + yellow('{:+.2f}%'.format((self.adjust-1) * 100))) yield '' @@ -879,7 +861,7 @@ class Ticker: yield self.table_hdr for row in self.rows: - if isinstance(row,str): + if isinstance(row, str): yield ('-' * self.hl_wid) else: try: @@ -896,21 +878,20 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym,amt in cfg.portfolio.items(): + for sym, amt in cfg.portfolio.items(): try: - yield self.fmt_row(self.data[sym],amt=amt) + yield self.fmt_row(self.data[sym], amt=amt) except KeyError: yield gray(f'(no data for {sym})') yield '-' * self.hl_wid if not cfg.btc_only: yield self.fs_num.format( lbl = 'TOTAL', pc3='', pc4='', pc1='', pc2='', upd='', amt='', - **{ k.replace('-','_'): v for k,v in self.prices['total'].items() } - ) + **{k.replace('-', '_'): v for k, v in self.prices['total'].items()}) class overview(base): - def __init__(self,data): + def __init__(self, data): self.data = data self.adjust = cfg.adjust self.show_adj = self.adjust != 1 @@ -922,25 +903,26 @@ class Ticker: self.format_last_update_col() if cfg.portfolio: - self.prices['total'] = { col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] - for row in self.rows if isinstance(row,tuple) and row.id in cfg.portfolio and row.id in data) - for col_id in self.col_ids } + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + for row in self.rows + if isinstance(row, tuple) and row.id in cfg.portfolio and row.id in data) + for col_id in self.col_ids} self.init_prec() self.init_fs() - def get_row_prices(self,id): + def get_row_prices(self, id): if id in self.data: d = self.data[id] - return { k: ( + return {k: ( d['price_btc'] if k == 'btc-bitcoin' else d['price_usd'] / self.col_usd_prices[k] - ) * self.adjust for k in self.col_ids } + ) * self.adjust for k in self.col_ids} - def fmt_row(self,d,amt=None,amt_fmt=None): + def fmt_row(self, d, amt=None, amt_fmt=None): def fmt_pct(n): - return gray(' --') if n is None else (red,green)[n>=0](f'{n:+7.2f}') + return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] @@ -957,53 +939,45 @@ class Ticker: pc4 = fmt_pct(d.get('percent_change_30d')), upd = d.get('last_updated_fmt'), amt = amt_fmt, - **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() } - ) + **{k.replace('-', '_'): v * (1 if amt is None else amt) for k, v in p.items()}) def init_fs(self): - col_prec = {'usd-us-dollar':2+cfg.add_prec,'btc-bitcoin':8+cfg.add_prec } # | self.uprec # Python 3.9 - col_prec.update(self.uprec) - col_wid = {'usd-us-dollar':8+cfg.add_prec,'btc-bitcoin':12+cfg.add_prec } # """ - col_wid.update(self.uwid) + col_prec = {'usd-us-dollar': 2+cfg.add_prec, 'btc-bitcoin': 8+cfg.add_prec} | self.uprec max_row = max( - ( (k,v['btc-bitcoin']) for k,v in self.prices.items() ), - key = lambda a: a[1] - ) - widths = { k: len('{:{}.{}f}'.format( self.prices[max_row[0]][k], self.comma, col_prec[k] )) - for k in self.col_ids } + ((k, v['btc-bitcoin']) for k, v in self.prices.items()), + key = lambda a: a[1]) + widths = {k: len('{:{}.{}f}'.format(self.prices[max_row[0]][k], self.comma, col_prec[k])) + for k in self.col_ids} - fd = namedtuple('format_str_data',['fs_str','fs_num','wid']) + fd = namedtuple('format_str_data', ['fs_str', 'fs_num', 'wid']) col_fs_data = { - 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid), + 'label': fd(f'{{lbl:{self.col1_wid}}}', f'{{lbl:{self.col1_wid}}}', self.col1_wid), 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), - 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2), - 'amt': fd(' {amt}', ' {amt}', 21), - } -# } | { k: fd( # Python 3.9 - col_fs_data.update({ k: fd( - ' {{{}:>{}}}'.format( k.replace('-','_'), widths[k] ), - ' {{{}:{}{}.{}f}}'.format( k.replace('-','_'), widths[k], self.comma, col_prec[k] ), - widths[k]+2 - ) for k in self.col_ids - }) + 'update_time': fd(' {upd}', ' {upd}', + max((19 if cfg.portfolio else 0), self.upd_w) + 2), + 'amt': fd(' {amt}', ' {amt}', 21) + } | {k: fd( + ' {{{}:>{}}}'.format(k.replace('-', '_'), widths[k]), + ' {{{}:{}{}.{}f}}'.format(k.replace('-', '_'), widths[k], self.comma, col_prec[k]), + widths[k] + 2 + ) for k in self.col_ids} cols = ( - ['label','usd-us-dollar'] + + ['label', 'usd-us-dollar'] + [asset.id for asset in self.usr_col_assets] + - [a for a,b in ( - ( 'btc-bitcoin', not cfg.btc_only ), - ( 'pct1y', 'y' in cfg.percent_cols ), - ( 'pct1m', 'm' in cfg.percent_cols ), - ( 'pct1w', 'w' in cfg.percent_cols ), - ( 'pct1d', 'd' in cfg.percent_cols ), - ( 'update_time', cfg.update_time ), - ) if b] - ) + [a for a, b in ( + ('btc-bitcoin', not cfg.btc_only), + ('pct1y', 'y' in cfg.percent_cols), + ('pct1m', 'm' in cfg.percent_cols), + ('pct1w', 'w' in cfg.percent_cols), + ('pct1d', 'd' in cfg.percent_cols), + ('update_time', cfg.update_time)) + if b]) cols2 = list(cols) if cfg.update_time: cols2.pop() @@ -1029,17 +1003,16 @@ class Ticker: amt = ' AMOUNT', usd_us_dollar = 'USD', btc_bitcoin = ' BTC', - **{ a.id.replace('-','_'): a.symbol for a in self.usr_col_assets } - ) + **{a.id.replace('-', '_'): a.symbol for a in self.usr_col_assets}) class trading(base): - def __init__(self,data): + def __init__(self, data): self.data = data self.asset = cfg.query.asset._replace(id=self.get_id(cfg.query.asset)) self.to_asset = ( cfg.query.to_asset._replace(id=self.get_id(cfg.query.to_asset)) - if cfg.query.to_asset else None ) + if cfg.query.to_asset else None) self.col_ids = [self.asset.id] self.adjust = cfg.adjust if self.to_asset: @@ -1048,12 +1021,13 @@ class Ticker: real_price = ( self.asset.amount * data[self.asset.id]['price_usd'] - / data[self.to_asset.id]['price_usd'] - ) + / data[self.to_asset.id]['price_usd']) if self.adjust != 1: - die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier') + die(1, + 'the --adjust option may not be combined with TO_AMOUNT ' + 'in the trade specifier') self.adjust = self.offer / real_price - self.hl_ids = [self.asset.id,self.to_asset.id] + self.hl_ids = [self.asset.id, self.to_asset.id] else: self.hl_ids = [self.asset.id] @@ -1070,10 +1044,10 @@ class Ticker: self.init_prec() self.init_fs() - def get_row_prices(self,id): + def get_row_prices(self, id): if id in self.data: d = self.data[id] - return { k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids } + return {k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids} def init_fs(self): self.max_wid = max( @@ -1081,10 +1055,8 @@ class Ticker: v[self.asset.id] * self.asset.amount, 16 + cfg.add_prec, self.comma, - 8 + cfg.add_prec - )) - for v in self.prices.values() - ) + 8 + cfg.add_prec)) + for v in self.prices.values()) self.fs_str = '{lbl:%s} {p_spot}' % self.col1_wid self.hl_wid = self.col1_wid + self.max_wid + 1 if self.show_adj: @@ -1094,20 +1066,19 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self,d): + def fmt_row(self, d): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount - p_spot = '{:{}{}.{}f}'.format( p, self.max_wid, self.comma, 8+cfg.add_prec ) + p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) p_adj = ( - '{:{}{}.{}f}'.format( p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec ) - if self.show_adj else '' ) + '{:{}{}.{}f}'.format(p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec) + if self.show_adj else '') return self.fs_str.format( lbl = self.create_label(id) if cfg.name_labels else d['symbol'], p_spot = green(p_spot) if id in self.hl_ids else p_spot, p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, - upd = d.get('last_updated_fmt'), - ) + upd = d.get('last_updated_fmt')) @property def table_hdr(self): @@ -1115,12 +1086,11 @@ class Ticker: lbl = '', p_spot = '{t:>{w}}'.format( t = 'SPOT PRICE', - w = self.max_wid ), + w = self.max_wid), p_adj = '{t:>{w}}'.format( t = ('OFFERED' if self.offer else 'ADJUSTED') + ' PRICE', - w = self.max_wid ), - upd = 'UPDATED' - ) + w = self.max_wid), + upd = 'UPDATED') @property def subhdr(self): @@ -1133,9 +1103,8 @@ class Ticker: ) + ( ( ' =>' + - (' {:{}}'.format(self.offer,self.comma) if self.offer else '') + + (' {:{}}'.format(self.offer, self.comma) if self.offer else '') + ' {} ({})'.format( self.to_asset.symbol, - self.create_label(self.to_asset.id) ) - ) if self.to_asset else '' ) - ) + self.create_label(self.to_asset.id)) + ) if self.to_asset else '')) diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 1d5426b..f36faad 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -21,53 +21,52 @@ mmgen_node_tools.Util: utility functions for MMGen node tools import time -def get_hms(t=None,utc=False,no_secs=False): +def get_hms(t=None, utc=False, no_secs=False): secs = t or time.time() - ret = (time.localtime,time.gmtime)[utc](secs) - fs,n = (('{:02}:{:02}:{:02}',6),('{:02}:{:02}',5))[no_secs] + ret = (time.localtime, time.gmtime)[utc](secs) + fs, n = (('{:02}:{:02}:{:02}', 6), ('{:02}:{:02}', 5))[no_secs] return fs.format(*ret[3:n]) -def get_day_hms(t=None,utc=False): +def get_day_hms(t=None, utc=False): secs = t or time.time() - ret = (time.localtime,time.gmtime)[utc](secs) + ret = (time.localtime, time.gmtime)[utc](secs) return '{:04}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*ret[0:6]) -def do_system(cmd,testing=False,shell=False): +def do_system(cmd, testing=False, shell=False): if testing: from mmgen.util import msg msg("Would execute: '%s'" % cmd) return True else: import subprocess - return subprocess.call((cmd if shell else cmd.split()),shell,stderr=subprocess.PIPE) + return subprocess.call((cmd if shell else cmd.split()), shell, stderr=subprocess.PIPE) -def get_url(url,gzip_ok=False,proxy=None,timeout=60,verbose=False,debug=False): +def get_url(url, gzip_ok=False, proxy=None, timeout=60, verbose=False, debug=False): if debug: print('get_url():') print(' url', url) - print(' gzip_ok:',gzip_ok, 'proxy:',proxy, 'timeout:',timeout, 'verbose:',verbose) - import pycurl,io + print(' gzip_ok:', gzip_ok, 'proxy:', proxy, 'timeout:', timeout, 'verbose:', verbose) + import pycurl, io c = pycurl.Curl() c_out = io.StringIO() - c.setopt(pycurl.WRITEFUNCTION,c_out.write) - c.setopt(pycurl.TIMEOUT,timeout) - c.setopt(pycurl.FOLLOWLOCATION,True) - c.setopt(pycurl.COOKIEFILE,'') - c.setopt(pycurl.VERBOSE,verbose) + c.setopt(pycurl.WRITEFUNCTION, c_out.write) + c.setopt(pycurl.TIMEOUT, timeout) + c.setopt(pycurl.FOLLOWLOCATION, True) + c.setopt(pycurl.COOKIEFILE, '') + c.setopt(pycurl.VERBOSE, verbose) if gzip_ok: - c.setopt(pycurl.USERAGENT,'Lynx/2.8.9dev.8 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.4.9') + c.setopt(pycurl.USERAGENT, 'Lynx/2.8.9dev.8 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.4.9') c.setopt(pycurl.HTTPHEADER, [ 'Accept: text/html, text/plain, text/sgml, text/css, application/xhtml+xml, */*;q=0.01', 'Accept-Encoding: gzip', - 'Accept-Language: en'] - ) + 'Accept-Language: en']) if proxy: - c.setopt(pycurl.PROXY,proxy) - c.setopt(pycurl.URL,url) + c.setopt(pycurl.PROXY, proxy) + c.setopt(pycurl.URL, url) c.perform() text = c_out.getvalue() if text[:2] == '\x1f\x8b': # gzip magic number - c_out.seek(0,0) + c_out.seek(0, 0) import gzip with gzip.GzipFile(fileobj=c_out) as f: text = f.read() @@ -103,20 +102,19 @@ big_digits = { """ } -_bnums_c,_bpunc_c = [[l.strip('\n') + ' ' * (big_digits[m]*big_digits['n']) +_bnums_c, _bpunc_c = [[l.strip('\n') + ' ' * (big_digits[m]*big_digits['n']) for l in big_digits[k][1:].split('\n')] - for k,m in (('nums','w'),('punc','pw'))] + for k, m in (('nums', 'w'), ('punc', 'pw'))] -_bnums_n,_bpunc_n = [[[l[0+(j*w):w+(j*w)] for l in i] - for j in range(big_digits[n])] for n,w,i in - (('n',big_digits['w'],_bnums_c),('pn',big_digits['pw'],_bpunc_c))] +_bnums_n, _bpunc_n = [[[l[0+(j*w):w+(j*w)] for l in i] + for j in range(big_digits[n])] for n, w, i in + (('n', big_digits['w'], _bnums_c), ('pn', big_digits['pw'], _bpunc_c))] -def display_big_digits(s,pre='',suf=''): - s = [int((d,10,11)[(d in '.:')+(d==':')]) for d in s] +def display_big_digits(s, pre='', suf=''): + s = [int((d, 10, 11)[(d in '.:')+(d==':')]) for d in s] return pre + ('\n'+pre).join( - [''.join([(_bnums_n+_bpunc_n)[d][l] for d in s]) + suf for l in range(big_digits['h'])] - ) + [''.join([(_bnums_n+_bpunc_n)[d][l] for d in s]) + suf for l in range(big_digits['h'])]) if __name__ == '__main__': num = '2345.17' - print(display_big_digits(num,pre='+ ',suf=' +')) + print(display_big_digits(num, pre='+ ', suf=' +')) diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 37f0482..4647029 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -16,7 +16,7 @@ import sys from mmgen.obj import CoinTxID from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,suf,make_timestr,async_run +from mmgen.util import msg, Msg, die, suf, make_timestr, async_run from mmgen.color import red opts_data = { @@ -32,16 +32,16 @@ opts_data = { } } -def do_output(proto,addr_data,blk_hdrs): +def do_output(proto, addr_data, blk_hdrs): col1w = len(str(len(addr_data))) indent = ' ' * (col1w + 2) - for n,(addr,unspents) in enumerate(addr_data.items(),1): + for n, (addr, unspents) in enumerate(addr_data.items(), 1): Msg(f'\n{n:{col1w}}) Address: {addr.hl(addr.view_pref)}') if unspents: - heights = { u['height'] for u in unspents } + heights = {u['height'] for u in unspents} Msg('{}Balance: {}'.format( indent, sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), @@ -50,22 +50,21 @@ def do_output(proto,addr_data,blk_hdrs): red(str(len(unspents))), suf(unspents), red(str(len(heights))), - suf(heights) )) + suf(heights))) blk_w = len(str(unspents[-1]['height'])) - fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent,max(5,blk_w)) - Msg(fs.format('Block','Date','TxID','Vout',' Amount')) + fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent, max(5, blk_w)) + Msg(fs.format('Block', 'Date', 'TxID', 'Vout', ' Amount')) for u in unspents: Msg(fs.format( u['height'], - make_timestr( blk_hdrs[u['height']]['time'] ), + make_timestr(blk_hdrs[u['height']]['time']), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(6, color=True, prec=8) - )) + proto.coin_amt(u['amount']).fmt(6, color=True, prec=8))) else: Msg(f'{indent}No balance') -def do_output_tabular(proto,addr_data,blk_hdrs): +def do_output_tabular(proto, addr_data, blk_hdrs): col1w = len(str(len(addr_data))) + 1 max_addrw = max(len(addr) for addr in addr_data) @@ -75,9 +74,9 @@ def do_output_tabular(proto,addr_data,blk_hdrs): lb_w = max(len(h) for h in lb_heights) fs = ( - ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w,max(5,fb_w),max(4,lb_w)) + ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w, max(5, fb_w), max(4, lb_w)) if cfg.first_block else - ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w,max(4,lb_w)) ) + ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w, max(4, lb_w))) Msg('\n' + fs.format( n = '', @@ -87,20 +86,19 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = 'Block', B = 'Last', T = 'Block', - A = ' Amount' )) + A = ' Amount')) - for n,(addr,unspents) in enumerate(addr_data.items(),1): + for n, (addr, unspents) in enumerate(addr_data.items(), 1): if unspents: Msg(fs.format( n = str(n) + ')', a = addr.fmt(addr.view_pref, max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], - t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), + t = make_timestr(blk_hdrs[unspents[0]['height']]['time']), B = unspents[-1]['height'], - T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8) - )) + T = make_timestr(blk_hdrs[unspents[-1]['height']]['time']), + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8))) else: Msg(fs.format( n = str(n) + ')', @@ -110,61 +108,61 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = '', B = '-', T = '', - A = ' -' )) + A = ' -')) async def main(req_addrs): proto = cfg._proto from mmgen.addr import CoinAddr - addrs = [CoinAddr(proto,addr) for addr in req_addrs] + addrs = [CoinAddr(proto, addr) for addr in req_addrs] from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,ignore_wallet=True) + rpc = await rpc_init(cfg, ignore_wallet=True) height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') from mmgen.proto.btc.misc import scantxoutset - res = await scantxoutset( cfg, rpc, [f'addr({addr})' for addr in addrs] ) + res = await scantxoutset(cfg, rpc, [f'addr({addr})' for addr in addrs]) if not res['success']: - die(1,'UTXO scanning failed or was interrupted') + die(1, 'UTXO scanning failed or was interrupted') elif not res['unspents']: msg('Address has no balance' if len(addrs) == 1 else - 'Addresses have no balances' ) + 'Addresses have no balances') else: addr_data = {k:[] for k in addrs} if 'desc' in res['unspents'][0]: import re - for unspent in sorted(res['unspents'],key=lambda x: x['height']): - addr = re.match('addr\((.*?)\)',unspent['desc'])[1] + for unspent in sorted(res['unspents'], key=lambda x: x['height']): + addr = re.match('addr\((.*?)\)', unspent['desc'])[1] addr_data[addr].append(unspent) else: from mmgen.proto.btc.tx.base import decodeScriptPubKey - for unspent in sorted(res['unspents'],key=lambda x: x['height']): + for unspent in sorted(res['unspents'], key=lambda x: x['height']): ds = decodeScriptPubKey(proto, unspent['scriptPubKey']) addr_data[ds.addr].append(unspent) good_addrs = len([v for v in addr_data.values() if v]) Msg('Total: {} in {} address{}'.format( - proto.coin_amt(res['total_amount']).hl2(unit=True,fs='{:,}'), + proto.coin_amt(res['total_amount']).hl2(unit=True, fs='{:,}'), red(str(good_addrs)), - suf(good_addrs,'es') - )) + suf(good_addrs, 'es'))) blk_heights = {i['height'] for i in res['unspents']} blk_hashes = await rpc.batch_call('getblockhash', [(h,) for h in blk_heights]) blk_hdrs = await rpc.batch_call('getblockheader', [(H,) for H in blk_hashes]) - (do_output_tabular if cfg.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) + (do_output_tabular if cfg.tabular else do_output)( + proto, addr_data, dict(zip(blk_heights, blk_hdrs))) -cfg = Config( opts_data=opts_data, init_opts={'rpc_backend':'aiohttp'} ) +cfg = Config(opts_data=opts_data, init_opts={'rpc_backend': 'aiohttp'}) if len(cfg._args) < 1: - die(1,'This command requires at least one coin address argument') + die(1, 'This command requires at least one coin address argument') try: async_run(cfg, main, args=[cfg._args]) diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 18136c4..eaed15b 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -20,9 +20,9 @@ mmnode-blocks-info: Display information about a block or range of blocks """ -from mmgen.cfg import gc,Config -from mmgen.util import async_run,fmt_list -from .BlocksInfo import BlocksInfo,JSONBlocksInfo +from mmgen.cfg import gc, Config +from mmgen.util import async_run, fmt_list +from .BlocksInfo import BlocksInfo, JSONBlocksInfo opts_data = { 'sets': [ @@ -145,14 +145,13 @@ EXAMPLES: $ {p} --rpc-backend=aio -H +1000 This program requires a txindex-enabled daemon for correct operation. -""" }, +"""}, 'code': { - 'notes': lambda cfg,proto,s: s.format( + 'notes': lambda cfg, proto, s: s.format( I = proto.diff_adjust_interval, - F = fmt_list(BlocksInfo.fields,fmt='bare'), - S = fmt_list(BlocksInfo.all_stats,fmt='bare'), - p = gc.prog_name, - ) + F = fmt_list(BlocksInfo.fields, fmt='bare'), + S = fmt_list(BlocksInfo.all_stats, fmt='bare'), + p = gc.prog_name) } } @@ -172,7 +171,7 @@ async def main(): await m.process_blocks() if m.last: - for i,sname in enumerate(m.stats): + for i, sname in enumerate(m.stats): m.process_stats_pre(i) await m.process_stats(sname) diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 3211f7f..188f60e 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -21,10 +21,10 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool """ from mmgen.cfg import Config -from mmgen.util import async_run,die,fmt,make_timestr,check_int_between -from mmgen.util2 import int2bytespec,parse_bytespec +from mmgen.util import async_run, die, fmt, make_timestr, check_int_between +from mmgen.util2 import int2bytespec, parse_bytespec -min_prec,max_prec,dfl_prec = (0,6,4) +min_prec, max_prec, dfl_prec = (0, 6, 4) fee_brackets = [ 1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, @@ -42,9 +42,9 @@ fee_brackets = [ opts_data = { 'sets': [ - ('detail',True,'ranges',True), - ('detail',True,'show_mb_col',True), - ('detail',True,'precision',6), + ('detail', True, 'ranges', True), + ('detail', True, 'show_mb_col', True), + ('detail', True, 'precision', 6), ], 'text': { 'desc': 'Visualize the fee structure of a node’s mempool', @@ -83,37 +83,37 @@ cfg = Config(opts_data=opts_data) if cfg.ignore_below: if cfg.show_empty: - die(1,'Conflicting options: --ignore-below, --show-empty') + die(1, 'Conflicting options: --ignore-below, --show-empty') ignore_below = parse_bytespec(cfg.ignore_below) precision = ( check_int_between(cfg.precision, min_prec, max_prec, desc='--precision arg') - if cfg.precision else dfl_prec ) + if cfg.precision else dfl_prec) from mmgen.term import get_terminal_size width = cfg.columns or get_terminal_size().width class fee_bracket: - def __init__(self,top,bottom): + def __init__(self, top, bottom): self.top = top self.bottom = bottom self.tx_bytes = 0 self.tx_bytes_cum = 0 self.skip = False -def log(data,fn): +def log(data, fn): import json from mmgen.rpc.util import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( cfg = cfg, outfile = fn, - data = json.dumps(data,cls=json_encoder,sort_keys=True,indent=4), + data = json.dumps(data, cls=json_encoder, sort_keys=True, indent=4), desc = 'mempool', - ask_overwrite = False ) + ask_overwrite = False) -def create_data(coin_amt,mempool): - out = [fee_bracket(fee_brackets[i],fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] +def create_data(coin_amt, mempool): + out = [fee_bracket(fee_brackets[i], fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] # populate fee brackets: size_key = 'size' if proto.coin == 'BCH' else 'vsize' @@ -143,7 +143,7 @@ def create_data(coin_amt,mempool): return out -def gen_header(host,mempool,blockcount): +def gen_header(host, mempool, blockcount): yield fmt(f""" Mempool Fee Structure @@ -159,27 +159,26 @@ def gen_header(host,mempool,blockcount): elif cfg.ignore_below: yield 'Ignoring fee brackets with less than {:,} bytes ({})'.format( ignore_below, - int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), - ) + int2bytespec(ignore_below, 'MB', '0.6', strip=True, add_space=True)) if cfg.include_current: yield 'Including transactions in current fee bracket in Total MB amounts' def fmt_mb(n): - return int2bytespec(n,'MB',f'0.{precision}',print_sym=False) + return int2bytespec(n, 'MB', f'0.{precision}', print_sym=False) def gen_body(data): - tx_bytes_max = max((i.tx_bytes for i in data),default=0) - top_max = max((i.top for i in data),default=0) - bot_max = max((i.bottom for i in data),default=0) - col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'),6) + tx_bytes_max = max((i.tx_bytes for i in data), default=0) + top_max = max((i.top for i in data), default=0) + bot_max = max((i.bottom for i in data), default=0) + col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'), 6) col2_w = len(fmt_mb(tx_bytes_max)) if cfg.show_mb_col else 0 col3_w = len(fmt_mb(data[-1].tx_bytes_cum)) if data else 0 col4_w = width - col1_w - col2_w - col3_w - (4 if col2_w else 3) if cfg.show_mb_col: - fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w,col2_w,col3_w) + fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w, col2_w, col3_w) else: - fs = '{a:<%i} {c:>%i} {d}' % (col1_w,col3_w) + fs = '{a:<%i} {c:>%i} {d}' % (col1_w, col3_w) yield fs.format(a='', b='', c=f'{"Total":<{col3_w}}', d='') yield fs.format(a='sat/B', b=f'{"MB":<{col2_w}}', c=f'{"MB":<{col3_w}}', d='') @@ -188,16 +187,16 @@ def gen_body(data): if not i.skip: cum_bytes = i.tx_bytes_cum + i.tx_bytes if cfg.include_current else i.tx_bytes_cum yield fs.format( - a = '{}-{}'.format(i.bottom,i.top) if cfg.ranges else i.top, + a = '{}-{}'.format(i.bottom, i.top) if cfg.ranges else i.top, b = fmt_mb(i.tx_bytes), c = fmt_mb(cum_bytes), - d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) ) + d = '-' * int(col4_w * (i.tx_bytes / tx_bytes_max))) yield fs.format( a = 'TOTAL', b = '', c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes if data else 0), - d = '' ) + d = '') async def main(): @@ -205,19 +204,19 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) - mempool = await c.call('getrawmempool',True) + mempool = await c.call('getrawmempool', True) if cfg.log: - log(mempool,'mempool.json') + log(mempool, 'mempool.json') - data = create_data(proto.coin_amt,mempool) + data = create_data(proto.coin_amt, mempool) cfg._util.stdout_or_pager( '\n'.join(gen_header( c.host, mempool, - await c.call('getblockcount') )) + '\n\n' + - '\n'.join(gen_body(data)) + '\n' ) + await c.call('getblockcount'))) + '\n\n' + + '\n'.join(gen_body(data)) + '\n') async_run(cfg, main) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 8f0a3a3..60c5440 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -28,7 +28,7 @@ from mmgen.util import async_run bdr_proj = 9.95 opts_data = { - 'sets': [('mined',True,'list',True)], + 'sets': [('mined', True, 'list', True)], 'text': { 'desc': 'Estimate date(s) of future block subsidy halving(s)', 'usage':'[opts]', @@ -41,7 +41,7 @@ opts_data = { {bdr_proj:.5f} min) -s, --sample-size=N Block range to calculate block discovery interval for next halving estimate (default: dynamically calculated) -""" } +"""} } cfg = Config(opts_data=opts_data) @@ -53,14 +53,14 @@ def date(t): return '{}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*time.gmtime(t)[:6]) def dhms(t): - t,neg = (-t,'-') if t < 0 else (t,' ') + t, neg = (-t, '-') if t < 0 else (t, ' ') return f'{neg}{t//60//60//24} days, {t//60//60%24:02}:{t//60%60:02}:{t%60:02} h/m/s' def time_diff_warning(t_diff): if abs(t_diff) > 60*60: print('Warning: block tip time is {} {} clock time!'.format( dhms(abs(t_diff)), - ('behind','ahead of')[t_diff<0])) + ('behind', 'ahead of')[t_diff<0])) async def main(): @@ -72,9 +72,9 @@ async def main(): tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' remaining = proto.halving_interval - tip % proto.halving_interval - sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1,max(remaining,144)) + sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1, max(remaining, 144)) - cur,old = await c.gathered_call('getblockstats',((tip,),(tip - sample_size,))) + cur, old = await c.gathered_call('getblockstats', ((tip,), (tip - sample_size,))) clock_time = int(time.time()) time_diff_warning(clock_time - cur['time']) @@ -98,8 +98,7 @@ async def main(): f'Current block discovery interval (over last {sample_size} blocks): {bdr/60:0.2f} min\n\n' f'Current clock time (UTC): {date(clock_time)}\n' f'Est. halving date (UTC): {date(t_next)}\n' - f'Est. time until halving: {dhms(cur["time"] + t_rem - clock_time)}' - ) + f'Est. time until halving: {dhms(cur["time"] + t_rem - clock_time)}') async def print_halvings(): halving_blocknums = [i*proto.halving_interval for i in range(proto.max_halvings+1)][1:] @@ -108,13 +107,13 @@ async def main(): nhist = len(hist_halvings) nSubsidy = int(proto.start_subsidy / proto.coin_amt.satoshi) - block0_hash = await c.call('getblockhash',0) - block0_date = (await c.call('getblock',block0_hash))['time'] + block0_hash = await c.call('getblockhash', 0) + block0_date = (await c.call('getblock', block0_hash))['time'] def gen_data(): total_mined = 0 date = block0_date - for n,blk in enumerate(halving_blocknums): + for n, blk in enumerate(halving_blocknums): mined = (nSubsidy >> n) * proto.halving_interval if n == 0: mined -= nSubsidy # subtract unspendable genesis block subsidy @@ -123,13 +122,11 @@ async def main(): bdi = ( (hist_halvings[n]['time'] - date) / (proto.halving_interval * 60) if n < nhist else bdr/60 if n == nhist - else bdr_proj - ) + else bdr_proj) date = ( hist_halvings[n]['time'] if n < nhist - else t_next + int((n - nhist) * halving_secs) - ) - yield ( n, sub, blk, mined, total_mined, bdi, date ) + else t_next + int((n - nhist) * halving_secs)) + yield (n, sub, blk, mined, total_mined, bdi, date) if sub == 0: break @@ -150,8 +147,7 @@ async def main(): e = 'BDI (mins)', f = 'SUBSIDY ({proto.coin})', g = f'MINED ({proto.coin})', - h = f'TOTAL MINED ({proto.coin})' - ) + h = f'TOTAL MINED ({proto.coin})') + '\n' + fs.format( a = '-' * 7, @@ -159,22 +155,20 @@ async def main(): c = '-' * 19, d = '-' * 2, e = '-' * 10, - f = '-' * 13, + f = '-' * 17, g = '-' * 17, - h = '-' * 17 - ) + h = '-' * 17) + '\n' + '\n'.join(fs.format( - a = n + 1, - b = blk, - c = date(t), - d = ' P' if n > nhist else '' if n < nhist else ' E', - e = f'{bdr:8.5f}', - f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), - g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), - h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) - ) for n, sub, blk, mined, total_mined, bdr, t in gen_data()) - ) + a = n + 1, + b = blk, + c = date(t), + d = ' P' if n > nhist else '' if n < nhist else ' E', + e = f'{bdr:8.5f}', + f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), + g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), + h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) + ) for n, sub, blk, mined, total_mined, bdr, t in gen_data())) if cfg.list: await print_halvings() diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index b273ac6..f5cf992 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -20,7 +20,7 @@ mmnode-netrate: Bitcoin daemon network rate monitor """ -import sys,time +import sys, time from mmgen.cfg import Config from mmgen.util import async_run @@ -32,39 +32,38 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -""" - } +"""} } cfg = Config(opts_data=opts_data) -ERASE_LINE,CUR_UP = '\033[K','\033[1A' +ERASE_LINE, CUR_UP = '\033[K', '\033[1A' async def main(): from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) async def get_data(): d = await c.call('getnettotals') - return [float(e) for e in (d['totalbytesrecv'],d['totalbytessent'],d['timemillis'])] + return [float(e) for e in (d['totalbytesrecv'], d['totalbytessent'], d['timemillis'])] - rs,ss,ts = (None,None,None) + rs, ss, ts = (None, None, None) while True: - r,s,t = await get_data() + r, s, t = await get_data() if rs is not None: sys.stderr.write( '\rrcvd: {:9.2f} kB/s\nsent: {:9.2f} kB/s '.format( (r-rs)/(t-ts), - (s-ss)/(t-ts) )) + (s-ss)/(t-ts))) time.sleep(2) if rs is not None: - sys.stderr.write('{}{}{}'.format(ERASE_LINE,CUR_UP,ERASE_LINE)) + sys.stderr.write('{}{}{}'.format(ERASE_LINE, CUR_UP, ERASE_LINE)) - rs,ss,ts = (r,s,t) + rs, ss, ts = (r, s, t) try: async_run(cfg, main) diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 35292ea..6a7cb38 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -27,8 +27,7 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -""" - } +"""} } from mmgen.cfg import Config @@ -37,9 +36,9 @@ cfg = Config(opts_data=opts_data) async def main(): from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,ignore_wallet=True) + rpc = await rpc_init(cfg, ignore_wallet=True) - from .PeerBlocks import BlocksDisplay,PeersDisplay + from .PeerBlocks import BlocksDisplay, PeersDisplay blocks = BlocksDisplay(cfg) peers = PeersDisplay(cfg) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 351cb1e..c36d886 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -208,29 +208,26 @@ Customize output by editing the file To add a portfolio, edit the file ~/{pf_cfg} -""" - }, +"""}, 'code': { 'options': lambda s: s.format( - dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), - ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), + dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), + ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, - pc = fmt_list(Ticker.percent_cols,fmt='bare'), - ), + pc = fmt_list(Ticker.percent_cols, fmt='bare')), 'notes': lambda s: s.format( - assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), - cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), - pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), + cfg = os.path.relpath(cfg_in.cfg_file, start=homedir), + pf_cfg = os.path.relpath(cfg_in.portfolio_file, start=homedir), al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), - fi = src_cls['fi'](), - ) + fi = src_cls['fi']()) } } import os -from mmgen.util import fmt_list,fmt_dict +from mmgen.util import fmt_list, fmt_dict from mmgen.cfg import Config from . import Ticker @@ -238,7 +235,7 @@ gcfg = Config(opts_data=opts_data, caller_post_init=True) Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, cfg_in, src_cls gcfg._post_init() diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 2411f36..0f9db5b 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -23,7 +23,7 @@ mmnode-txfind: Find a transaction in the blockchain or mempool import sys from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,is_hex_str,async_run +from mmgen.util import msg, Msg, die, is_hex_str, async_run opts_data = { 'text': { @@ -48,29 +48,26 @@ msg_data = { 'normal': { 'none': 'Transaction not found in blockchain or mempool', 'block': 'Transaction is in block {b} ({c} confirmations)', - 'mem': 'Transaction is in mempool', - }, + 'mem': 'Transaction is in mempool'}, 'quiet': { 'none': 'None', 'block': '{b} {c}', - 'mem': 'mempool', - } -} + 'mem': 'mempool'}} async def main(txid): if len(txid) != 64 or not is_hex_str(txid): - die(2,f'{txid}: invalid transaction ID') + die(2, f'{txid}: invalid transaction ID') if cfg.verbose: msg(f'TxID: {txid}') from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) exitval = 0 try: tip1 = await c.call('getblockcount') - ret = await c.call('getrawtransaction',txid,True) + ret = await c.call('getrawtransaction', txid, True) tip2 = await c.call('getblockcount') except: Msg('\r' + msgs['none']) @@ -90,6 +87,6 @@ cfg = Config(opts_data=opts_data) msgs = msg_data['quiet' if cfg.quiet else 'normal'] if len(cfg._args) != 1: - die(1,'One transaction ID must be specified') + die(1, 'One transaction ID must be specified') sys.exit(async_run(cfg, main, args=[cfg._args[0]])) diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index 4792dee..a504678 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -335,19 +335,19 @@ class CmdTestRegtest(CmdTestBase): us = await r.rpc_call('listunspent',wallet='miner') tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') - outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} + outputs = {p.addr: tx1_amt for p in pairs[:nTxs]} outputs.update({burn_addr: self.proto.coin_amt(tx_input['amount']) - (tx1_amt*nTxs) - fee}) return await do_tx( - [{ 'txid': tx_input['txid'], 'vout': 0 }], + [{'txid': tx_input['txid'], 'vout': 0}], outputs, await r.miner_wif) async def do_tx2(tx,pairno): fee = self.proto.coin_amt(fees[pairno], from_decimal=True) - outputs = {p.addr:tx2_amt for p in pairs} + outputs = {p.addr: tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) return await do_tx( - [{ 'txid': tx['txid'], 'vout': pairno }], + [{'txid': tx['txid'], 'vout': pairno}], outputs, pairs[pairno].wif ) diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py index 7c2fdf3..444ae08 100644 --- a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -294,15 +294,14 @@ class fake_data: 20 7303 7307 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 21 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 22 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 7398 7409 - """ - } + """} def make_data(): def gen_address_data(): for line in fake_data.addresses.strip().split('\n'): data = line.split(maxsplit=2) - yield (data[0], {k:v for k,v in zip(('id','addr','subver'),data)}) + yield (data[0], {k: v for k, v in zip(('id', 'addr', 'subver'), data)}) def gen_iterations_data(): for line in fake_data.iterations.strip().split('\n'): @@ -320,8 +319,7 @@ class fake_data: 'id': int(d['id']), 'addr': d['addr'], 'subver': d['subver'], - 'inflight': [int(n)+830000 for n in blocks[iter_no]], - } + 'inflight': [int(n)+830000 for n in blocks[iter_no]]} def gen_data(): for iter_no in iterations_data: From b9957f918298329ab9057a94946abccdaad9d038 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 09:57:02 +0000 Subject: [PATCH 083/108] mmnode-ticker: support `--http-timeout` option --- mmgen_node_tools/Ticker.py | 4 ++++ mmgen_node_tools/data/version | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index ebefcf3..caee658 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -75,6 +75,7 @@ class DataSource: ['--compressed'], # adds 'Accept-Encoding: gzip' ['--proxy', cfg.proxy, isinstance(cfg.proxy, str)], ['--silent', not cfg.verbose], + ['--connect-timeout', str(gcfg.http_timeout), gcfg.http_timeout], [self.api_url]) if gcfg.testing: @@ -290,6 +291,9 @@ class DataSource: if gcfg.test_suite: kwargs.update({'timeout': 1, 'retry': 0}) + if gcfg.http_timeout: + kwargs.update({'timeout': gcfg.http_timeout}) + if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( self.symbols, diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 9f32e9d..6f735af 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev1 +3.6.dev2 From 06340ed521de5ba0333e2d4bf20105cde60b0e8c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 12:44:54 +0000 Subject: [PATCH 084/108] restore Nix support --- mmgen_node_tools/data/version | 2 +- nix/curl-cffi.nix | 50 +++++++++++++++++++++++++++++++ nix/node-tools-packages.nix | 4 +-- nix/use-system-libs.patch | 23 ++++++++++++++ nix/yahooquery-noversioning.patch | 45 ---------------------------- nix/yahooquery.nix | 39 ++++++++++-------------- 6 files changed, 92 insertions(+), 71 deletions(-) create mode 100644 nix/curl-cffi.nix create mode 100644 nix/use-system-libs.patch delete mode 100644 nix/yahooquery-noversioning.patch diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6f735af..f9d163c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev2 +3.6.dev3 diff --git a/nix/curl-cffi.nix b/nix/curl-cffi.nix new file mode 100644 index 0000000..15f9764 --- /dev/null +++ b/nix/curl-cffi.nix @@ -0,0 +1,50 @@ +{ + stdenv, + lib, + python, + buildPythonPackage, + fetchPypi, + curl-impersonate-chrome, +}: + +buildPythonPackage rec { + pname = "curl-cffi"; + # version = "0.13.0"; # uses option PROXY_CREDENTIAL_NO_REUSE, unavailable in current libcurl + version = "0.10.0"; + pyproject = true; + + src = fetchPypi { + pname = "curl_cffi"; + version = version; + # hash = "sha256-YuzZCjgr1QI3UONgbgqnyxo6i6QcFCcLjl4Unr9yxco="; # 0.13.0 + hash = "sha256-PjezUmjKWEkvVO0CCuS1DDPuDeutQUXbn3RvBO1GbrA="; # 0.10.0 + }; + + patches = [ ./use-system-libs.patch ]; + + buildInputs = [ curl-impersonate-chrome ]; + + build-system = with python.pkgs; [ + cffi + setuptools + ]; + + dependencies = with python.pkgs; [ + cffi + certifi + typing-extensions + ]; + + env = lib.optionalAttrs stdenv.cc.isGNU { + NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types"; + }; + + pythonImportsCheck = [ "curl_cffi" ]; + + meta = with lib; { + description = "Python binding for curl-impersonate via cffi"; + homepage = "https://curl-cffi.readthedocs.io"; + license = licenses.mit; + maintainers = with maintainers; [ chuangzhu ]; + }; +} diff --git a/nix/node-tools-packages.nix b/nix/node-tools-packages.nix index 19792c7..1244425 100644 --- a/nix/node-tools-packages.nix +++ b/nix/node-tools-packages.nix @@ -6,7 +6,7 @@ }; python-packages = with python.pkgs; { - yahooquery = (pkgs.callPackage ./yahooquery.nix {}); # ticker - pyyaml = pyyaml; # ticker + yahooquery = (callPackage ./yahooquery.nix {}); # ticker + pyyaml = pyyaml; # ticker }; } diff --git a/nix/use-system-libs.patch b/nix/use-system-libs.patch new file mode 100644 index 0000000..1d04487 --- /dev/null +++ b/nix/use-system-libs.patch @@ -0,0 +1,23 @@ +diff --git a/scripts/build.py b/scripts/build.py +index b705a0d..9bfcaab 100644 +--- a/scripts/build.py ++++ b/scripts/build.py +@@ -105,7 +105,6 @@ def get_curl_libraries(): + ffibuilder = FFI() + system = platform.system() + root_dir = Path(__file__).parent.parent +-download_libcurl() + + + ffibuilder.set_source( +@@ -114,9 +113,7 @@ ffibuilder.set_source( + #include "shim.h" + """, + # FIXME from `curl-impersonate` +- libraries=get_curl_libraries(), +- extra_objects=get_curl_archives(), +- library_dirs=[arch["libdir"]], ++ libraries=["curl-impersonate-chrome"], + source_extension=".c", + include_dirs=[ + str(root_dir / "include"), diff --git a/nix/yahooquery-noversioning.patch b/nix/yahooquery-noversioning.patch deleted file mode 100644 index 4bd3297..0000000 --- a/nix/yahooquery-noversioning.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/pyproject.toml b/pyproject.toml -index 9d3fb29..399c215 100644 ---- a/pyproject.toml -+++ b/pyproject.toml -@@ -10,24 +10,24 @@ readme = "README.md" - - [tool.poetry.dependencies] - python = ">=3.8.1,<4.0" --requests = "^2.31.0" --pandas = "^2.0.3" --requests-futures = "^1.0.1" --tqdm = "^4.65.0" --lxml = "^4.9.3" --selenium = {version = "^4.10.0", optional = true} --beautifulsoup4 = "^4.12.2" -+requests = ">=2.31.0" -+pandas = ">=2.0.3" -+requests-futures = ">=1.0.1" -+tqdm = ">=4.65.0" -+lxml = ">=4.9.3" -+selenium = {version = ">=4.10.0", optional = true} -+beautifulsoup4 = ">=4.12.2" - - [tool.poetry.dev-dependencies] --pytest = "^7.4.0" --isort = "^5.0.0" --flake8 = "^6.0.0" --mypy = "^1.4.1" --pytest-cov = "^4.1.0" --black = "^23.7.0" --pre-commit = "^3.3.3" --ipython = "^8.0.0" --mkdocs-material = "^9.1.18" -+pytest = ">=7.4.0" -+isort = ">=5.0.0" -+flake8 = ">=6.0.0" -+mypy = ">=1.4.1" -+pytest-cov = ">=4.1.0" -+black = ">=23.7.0" -+pre-commit = ">=3.3.3" -+ipython = ">=8.0.0" -+mkdocs-material = ">=9.1.18" - - [build-system] - requires = ["poetry-core>=1.0.0"] diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix index 8b1535d..3e8925f 100644 --- a/nix/yahooquery.nix +++ b/nix/yahooquery.nix @@ -1,37 +1,30 @@ { lib, - pkgs, - fetchFromGitHub, + buildPythonPackage, + fetchPypi, + python, }: -with pkgs.python312.pkgs; - buildPythonPackage rec { pname = "yahooquery"; - version = "2.3.7"; + version = "2.4.1"; pyproject = true; - disabled = pythonOlder "3.8.1"; - - src = fetchFromGitHub { - owner = "dpguthrie"; - repo = "yahooquery"; - rev = "refs/tags/v${version}"; - hash = "sha256-Iyuni1SoTB6f7nNFhN5A8Gnv9kV78frjpqvvW8qd+/M="; + src = fetchPypi { + pname = "yahooquery"; + version = version; + hash = "sha256-GQPGXq5qEtlelFAGNHkhbAeEbwE7riojkXkTUxt/rls="; }; - patches = [ ./yahooquery-noversioning.patch ]; + build-system = with python.pkgs; [ hatchling ]; - build-system = [ poetry-core ]; - - dependencies = [ - requests # ^2.31.0 - pandas # ^2.0.3 - requests-futures # ^1.0.1 - tqdm # ^4.65.0 - lxml # ^4.9.3 - selenium # {version = ^4.10.0, optional = true} - beautifulsoup4 # ^4.12.2 + propagatedBuildInputs = with python.pkgs; [ + (callPackage ./curl-cffi.nix {}) # >=0.10.0 + pandas + requests-futures + tqdm + lxml + beautifulsoup4 ]; doCheck = false; # skip tests From de833d75afb3fa313f2541eb1a71819a9c1ae141 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:47 +0000 Subject: [PATCH 085/108] whitespace, variable renames --- mmgen_node_tools/Ticker.py | 6 ++++-- test/cmdtest_d/misc.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index caee658..d681a81 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -192,8 +192,10 @@ class DataSource: @property def api_url(self): return ( - f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else - f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else + f'https://{self.api_host}/v1/tickers/btc-bitcoin' + if cfg.btc_only else + f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' + if self.asset_limit else f'https://{self.api_host}/v1/tickers') @property diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9528b76..fb48282 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -106,10 +106,17 @@ class CmdTestScripts(CmdTestBase): shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' - def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): + def ticker( + self, + args = [], + expect_list = None, + cached_data = True, + exit_val = None): t = self.spawn( 'mmnode-ticker', - (['--cached-data'] if cached else []) + self.ticker_args + args, + (['--cached-data'] if cached_data else []) + + self.ticker_args + + args, exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) @@ -121,7 +128,7 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False) + t = self.ticker(cached_data=False) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') @@ -137,7 +144,6 @@ class CmdTestScripts(CmdTestBase): 'BTC 23250.77 1.00000000 ETH 1659.66 0.07138094' ]) - def ticker4(self): return self.ticker( ['--widest','--add-columns=eurusd=x,inr-indian-rupee:79.5'], From 313c7af4bbf165ed65e5acd56e2461663d54bec2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 086/108] mmgen-ticker: test caching JSON data --- MANIFEST.in | 1 + mmgen_node_tools/Ticker.py | 12 ++++++--- test/cmdtest_d/httpd/ticker.py | 27 +++++++++++++++++++ test/cmdtest_d/misc.py | 27 ++++++++++++++----- test/init.sh | 1 + .../fakemods/mmgen_node_tools/Ticker.py | 23 ++++++++++++++++ 6 files changed, 82 insertions(+), 9 deletions(-) create mode 100755 test/cmdtest_d/httpd/ticker.py create mode 100644 test/overlay/fakemods/mmgen_node_tools/Ticker.py diff --git a/MANIFEST.in b/MANIFEST.in index 77b53ac..2581100 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,5 +7,6 @@ include test/test-release.d/*.sh include test/modtest_d/*.py include test/cmdtest_d/*.py include test/cmdtest_d/include/cfg.py +include test/cmdtest_d/httpd/ticker.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index d681a81..bfe899b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -175,6 +175,7 @@ class DataSource: desc = 'CoinPaprika' data_desc = 'cryptocurrency data' api_host = 'api.coinpaprika.com' + api_proto = 'https' ratelimit = 240 btc_ratelimit = 10 net_data_type = 'json' @@ -192,11 +193,11 @@ class DataSource: @property def api_url(self): return ( - f'https://{self.api_host}/v1/tickers/btc-bitcoin' + f'{self.api_proto}://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else - f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' + f'{self.api_proto}://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else - f'https://{self.api_host}/v1/tickers') + f'{self.api_proto}://{self.api_host}/v1/tickers') @property def json_fn(self): @@ -697,6 +698,11 @@ def make_cfg(gcfg_arg): cmd_args = gcfg._args cfg_in = get_cfg_in() + if gcfg.test_suite: # required for testing with overlay + from . import Ticker as this_mod + this_mod.src_cls = src_cls + this_mod.cfg_in = cfg_in + usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None diff --git a/test/cmdtest_d/httpd/ticker.py b/test/cmdtest_d/httpd/ticker.py new file mode 100755 index 0000000..2d946ae --- /dev/null +++ b/test/cmdtest_d/httpd/ticker.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# MMGen Node Tools, terminal-based programs for Bitcoin and forkcoin nodes +# Copyright (C)2013-2025 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +test.cmdtest_d.httpd.ticker: Ticker WSGI http server +""" + +from . import HTTPD + +class TickerServer(HTTPD): + name = 'ticker server' + port = 19900 + content_type = 'application/json' + + def make_response_body(self, method, environ): + + with open(f'test/ref/ticker/ticker.json') as fh: + text = fh.read() + + return text.encode() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index fb48282..e9d4c32 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -16,6 +16,7 @@ import os, shutil from ..include.common import cfg from .base import CmdTestBase +from .httpd.ticker import TickerServer refdir = os.path.join('test','ref','ticker') @@ -70,8 +71,9 @@ class CmdTestScripts(CmdTestBase): ), 'ticker': ( "'mmnode-ticker' script", - ('ticker1', 'ticker [--help)'), + ('ticker1', 'ticker [--help]'), ('ticker2', 'ticker (bad proxy)'), + ('ticker2a', 'ticker [--download=cc]'), ('ticker3', 'ticker [--cached-data]'), ('ticker4', 'ticker [--cached-data --wide]'), ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), @@ -90,9 +92,12 @@ class CmdTestScripts(CmdTestBase): ) } - @property - def ticker_args(self): - return [ f'--cachedir={self.tmpdir}', '--proxy=http://asdfzxcv:32459' ] + def __init__(self, cfg, trunner, cfgs, spawn): + if not trunner: + return + self.ticker_server = TickerServer(cfg) + self.ticker_server.start() + return super().__init__(cfg, trunner, cfgs, spawn) @property def nt_datadir(self): @@ -100,7 +105,6 @@ class CmdTestScripts(CmdTestBase): def ticker_setup(self): self.spawn('',msg_only=True) - shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) @@ -111,11 +115,15 @@ class CmdTestScripts(CmdTestBase): args = [], expect_list = None, cached_data = True, + add_opts = [], + use_proxy = True, exit_val = None): t = self.spawn( 'mmnode-ticker', (['--cached-data'] if cached_data else []) - + self.ticker_args + + [f'--cachedir={self.tmpdir}'] + + (['--proxy=http://asdfzxcv:32459'] if use_proxy else []) + + add_opts + args, exit_val = exit_val) if expect_list: @@ -136,6 +144,13 @@ class CmdTestScripts(CmdTestBase): t.exit_val = 1 if ret else 3 return t + def ticker2a(self): + t = self.ticker( + add_opts = ['--proxy', '', '--download=cc'], + cached_data = False, + use_proxy = False) + return t + def ticker3(self): return self.ticker( [], diff --git a/test/init.sh b/test/init.sh index 134f4f3..eec6ec3 100755 --- a/test/init.sh +++ b/test/init.sh @@ -98,6 +98,7 @@ create_test_links() { test/modtest.py hard test/test-release.sh symbolic test/cmdtest_d/base.py symbolic + test/cmdtest_d/httpd/__init__.py symbolic test/cmdtest_d/include/common.py symbolic test/cmdtest_d/include/runner.py symbolic test/cmdtest_d/include/group_mgr.py symbolic diff --git a/test/overlay/fakemods/mmgen_node_tools/Ticker.py b/test/overlay/fakemods/mmgen_node_tools/Ticker.py new file mode 100644 index 0000000..8d1ede4 --- /dev/null +++ b/test/overlay/fakemods/mmgen_node_tools/Ticker.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# MMGen Node Tools, terminal-based programs for Bitcoin and forkcoin nodes +# Copyright (C)2013-2025 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +fakemods.mmgen_node_tools.Ticker: fake module for Ticker class +""" + +from .Ticker_orig import * + +class overlay_fake_DataSource: + class coinpaprika: + api_host = 'localhost:19900' + api_proto = 'http' + +DataSource.coinpaprika.api_host = overlay_fake_DataSource.coinpaprika.api_host +DataSource.coinpaprika.api_proto = overlay_fake_DataSource.coinpaprika.api_proto From a8adef0be54c752c10a8f34d594ed3faaa23de0c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 087/108] mmgen-ticker: various fixes and cleanups --- mmgen_node_tools/Ticker.py | 39 +++++++++++++++++++--------------- test/cmdtest_d/httpd/ticker.py | 2 +- test/cmdtest_d/misc.py | 36 +++++++++++++++---------------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bfe899b..9aee86c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen +from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -99,18 +99,21 @@ class DataSource: if not os.path.exists(cfg.cachedir): os.makedirs(cfg.cachedir) - if not os.path.exists(self.json_fn): - open(self.json_fn, 'w').write('{}') - use_cached_data = cfg.cached_data and not gcfg.download if use_cached_data: data_type = 'json' - data_in = open(self.json_fn).read() + try: + data_in = open(self.json_fn).read() + except FileNotFoundError: + die(1, f'Cannot use cached data, because {self.json_fn_disp} does not exist') else: data_type = self.net_data_type - elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) - if elapsed >= self.timeout or gcfg.testing: + try: + mtime = os.stat(self.json_fn).st_mtime + except FileNotFoundError: + mtime = 0 + if (elapsed := int(time.time() - mtime)) >= self.timeout or gcfg.testing: if gcfg.testing: msg('') self.fetch_delay() @@ -148,14 +151,14 @@ class DataSource: if use_cached_data: if not cfg.quiet: - msg(f'Using cached data from ~/{self.json_fn_rel}') + msg(f'Using cached data from {self.json_fn_disp}') else: if os.path.exists(self.json_fn): os.rename(self.json_fn, self.json_fn + '.bak') with open(self.json_fn, 'w') as fh: fh.write(json_text) if not cfg.quiet: - msg(f'JSON data cached to ~/{self.json_fn_rel}') + msg(f'JSON data cached to {self.json_fn_disp}') if gcfg.download: sys.exit(0) @@ -168,8 +171,8 @@ class DataSource: return data @property - def json_fn_rel(self): - return os.path.relpath(self.json_fn, start=homedir) + def json_fn_disp(self): + return '~/' + os.path.relpath(self.json_fn, start=homedir) class coinpaprika(base): desc = 'CoinPaprika' @@ -186,8 +189,9 @@ class DataSource: self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self, elapsed): + rem = self.timeout - elapsed return ( - f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + + f'Rate limit exceeded! Retry in {rem} second{suf(rem)}' + ('' if cfg.btc_only else ', or use --cached-data or --btc')) @property @@ -270,7 +274,8 @@ class DataSource: 'last_updated': data['regularMarketTime']} def rate_limit_errmsg(self, elapsed): - return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' + rem = self.timeout - elapsed + return f'Rate limit exceeded! Retry in {rem} second{suf(rem)}, or use --cached-data' @property def json_fn(self): @@ -364,10 +369,10 @@ def gen_data(data): checking for duplicates. """ - def dup_sym_errmsg(dup_sym): + def dup_sym_errmsg(data_type, dup_sym): return ( f'The symbol {dup_sym!r} is shared by the following assets:\n' + - '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + + '\n ' + '\n '.join(d['id'] for d in data[data_type].data if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + f'instead of {dup_sym!r}') @@ -421,7 +426,7 @@ def gen_data(data): if not isinstance(v, dict): die(2, str(v)) if id in found['id']: - die(1, dup_sym_errmsg(id)) + die(1, dup_sym_errmsg('fi', id)) if m := data['hi'].get(k): spot = v['regularMarketPrice']['raw'] hist = tuple(m.values()) @@ -444,7 +449,7 @@ def gen_data(data): if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: - die(1, dup_sym_errmsg(d[k])) + die(1, dup_sym_errmsg('cc', d[k])) if not 'price_usd' in d: d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd diff --git a/test/cmdtest_d/httpd/ticker.py b/test/cmdtest_d/httpd/ticker.py index 2d946ae..6131875 100755 --- a/test/cmdtest_d/httpd/ticker.py +++ b/test/cmdtest_d/httpd/ticker.py @@ -21,7 +21,7 @@ class TickerServer(HTTPD): def make_response_body(self, method, environ): - with open(f'test/ref/ticker/ticker.json') as fh: + with open('test/ref/ticker/ticker.json') as fh: text = fh.read() return text.encode() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index e9d4c32..1cde41e 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -61,19 +61,16 @@ class CmdTestScripts(CmdTestBase): color = True cmd_group_in = ( - ('subgroup.ticker_setup', []), - ('subgroup.ticker', ['ticker_setup']), + ('subgroup.ticker', []), ) cmd_subgroups = { - 'ticker_setup': ( - "setup for 'ticker' subgroup", - ('ticker_setup', 'ticker setup'), - ), 'ticker': ( "'mmnode-ticker' script", ('ticker1', 'ticker [--help]'), + ('copy_files', 'copying JSON files to cache'), + ('ticker1a', 'ticker [--download=cc] (early caching)'), + ('ticker1b', 'ticker [--download=cc] (late caching)'), ('ticker2', 'ticker (bad proxy)'), - ('ticker2a', 'ticker [--download=cc]'), ('ticker3', 'ticker [--cached-data]'), ('ticker4', 'ticker [--cached-data --wide]'), ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), @@ -103,7 +100,7 @@ class CmdTestScripts(CmdTestBase): def nt_datadir(self): return os.path.join( cfg.data_dir_root, 'node_tools' ) - def ticker_setup(self): + def copy_files(self): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) @@ -135,20 +132,23 @@ class CmdTestScripts(CmdTestBase): t.expect('USAGE:') return t - def ticker2(self): - t = self.ticker(cached_data=False) - if not cfg.skipping_deps: - t.expect('Creating') - t.expect('Creating') - ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) - t.exit_val = 1 if ret else 3 - return t - - def ticker2a(self): + def ticker1a(self, first_run=True): t = self.ticker( add_opts = ['--proxy', '', '--download=cc'], cached_data = False, use_proxy = False) + if first_run and not cfg.skipping_deps: + t.expect('Creating') + t.expect('Creating') + return t + + def ticker1b(self): + return self.ticker1a(first_run=False) + + def ticker2(self): + t = self.ticker(cached_data=False) + ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) + t.exit_val = 1 if ret else 3 return t def ticker3(self): From 3b8aede9bad83e4187e888d7be939ae909508075 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 088/108] Ticker.gen_data(): new `process_data` class --- mmgen_node_tools/Ticker.py | 135 ++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9aee86c..8580553 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf +from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -389,6 +389,73 @@ def gen_data(data): if error: die(1, 'Missing data, exiting') + class process_data: + + def cc(): + nonlocal btcusd + for d in data['cc']: + if d['id'] == 'btc-bitcoin': + btcusd = Decimal(str(d['quotes']['USD']['price'])) + break + else: + raise ValueError('malformed cryptocurrency data') + for k in ('id', 'symbol'): + for d in data['cc']: + if wants[k]: + if d[k] in wants[k]: + if d[k] in found[k]: + die(1, dup_sym_errmsg('cc', d[k])) + if not 'price_usd' in d: + d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) + d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd + d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] + d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] + d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] + d['last_updated'] = int(datetime.datetime.fromisoformat( + d['last_updated']).timestamp()) + yield (d['id'], d) + found[k].add(d[k]) + wants[k].remove(d[k]) + if d[k] in usr_rate_assets_want[k]: + rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key + else: + break + + def fi(): + get_id = src_cls['fi'].get_id + conv_func = src_cls['fi'].conv_data + for k, v in data['fi'].items(): + id = get_id(k, v) + if wants['id']: + if id in wants['id']: + if not isinstance(v, dict): + die(2, str(v)) + if id in found['id']: + die(1, dup_sym_errmsg('fi', id)) + if hist := hist_close.get(k): + spot = v['regularMarketPrice']['raw'] + v['pct_chg_1wk'] = (spot / hist.close_1wk - 1) * 100 + v['pct_chg_4wks'] = (spot / hist.close_4wks - 1) * 100 # 4 weeks ≈ 1 month + v['pct_chg_1y'] = (spot / hist.close_1y - 1) * 100 + else: + v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None + yield (id, conv_func(id, v, btcusd)) + found['id'].add(id) + wants['id'].remove(id) + if id in usr_rate_assets_want['id']: # NB: using symbol instead of ID for key: + rate_assets[k] = conv_func(id, v, btcusd) + else: + break + + def hi(): + ret = namedtuple('historical_closing_prices', ['close_1wk', 'close_4wks', 'close_1y']) + nonlocal hist_close + for k, v in data['hi'].items(): + hist = tuple(v.values()) + hist_close[k] = ret(hist[-2]['close'], hist[-5]['close'], hist[0]['close']) + return () + rows_want = { 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} @@ -411,62 +478,20 @@ def gen_data(data): wants = {k: rows_want[k] | usr_wants[k] for k in ('id', 'symbol')} - for d in data['cc']: - if d['id'] == 'btc-bitcoin': - btcusd = Decimal(str(d['quotes']['USD']['price'])) - break + btcusd = Decimal('1') # dummy + hist_close = {} - get_id = src_cls['fi'].get_id - conv_func = src_cls['fi'].conv_data + parse_fail = False + for data_type in ('cc', 'hi', 'fi'): # 'fi' depends on 'cc' and 'hi' so must go last + if data_type in data: + try: + yield from getattr(process_data, data_type)() + except Exception as e: + rmsg(f'Error in source data {data_type!r}: {e}') + parse_fail = True - for k, v in data['fi'].items(): - id = get_id(k, v) - if wants['id']: - if id in wants['id']: - if not isinstance(v, dict): - die(2, str(v)) - if id in found['id']: - die(1, dup_sym_errmsg('fi', id)) - if m := data['hi'].get(k): - spot = v['regularMarketPrice']['raw'] - hist = tuple(m.values()) - v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( - (spot / hist[-2]['close'] - 1) * 100, - (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month - (spot / hist[0]['close'] - 1) * 100) - else: - v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None - yield (id, conv_func(id, v, btcusd)) - found['id'].add(id) - wants['id'].remove(id) - if id in usr_rate_assets_want['id']: - rate_assets[k] = conv_func(id, v, btcusd) # NB: using symbol instead of ID for key - else: - break - - for k in ('id', 'symbol'): - for d in data['cc']: - if wants[k]: - if d[k] in wants[k]: - if d[k] in found[k]: - die(1, dup_sym_errmsg('cc', d[k])) - if not 'price_usd' in d: - d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) - d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd - d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] - d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] - d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] - d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] - # .replace('Z','+00:00') -- Python 3.9 backport - d['last_updated'] = int(datetime.datetime.fromisoformat( - d['last_updated'].replace('Z', '+00:00')).timestamp()) - yield (d['id'], d) - found[k].add(d[k]) - wants[k].remove(d[k]) - if d[k] in usr_rate_assets_want[k]: - rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key - else: - break + if parse_fail: + die(2, 'Invalid data encountered, exiting') check_assets_found(usr_wants, found) From e7de6890796505f2fa9499fae80c459455dc7aa2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 089/108] mmnode-ticker: cache network data after parsing --- mmgen_node_tools/Ticker.py | 48 ++++++++++++++++++++++++----------- mmgen_node_tools/data/version | 2 +- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 8580553..bbca8ef 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -19,7 +19,7 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys, os, re, time, datetime, json, yaml, random +import os, re, time, datetime, json, yaml, random from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple @@ -149,20 +149,18 @@ class DataSource: elif 'error' in data: die(1, data['error']) + self.data = self.postprocess_data(data) + if use_cached_data: + self.json_text = None if not cfg.quiet: msg(f'Using cached data from {self.json_fn_disp}') else: - if os.path.exists(self.json_fn): - os.rename(self.json_fn, self.json_fn + '.bak') - with open(self.json_fn, 'w') as fh: - fh.write(json_text) - if not cfg.quiet: - msg(f'JSON data cached to {self.json_fn_disp}') - if gcfg.download: - sys.exit(0) + self.json_text = json_text + if cache_data(self, no_overwrite=True): + self.json_text = None - return self.postprocess_data(data) + return self def json_data_error_msg(self, json_text): pass @@ -393,14 +391,14 @@ def gen_data(data): def cc(): nonlocal btcusd - for d in data['cc']: + for d in data['cc'].data: if d['id'] == 'btc-bitcoin': btcusd = Decimal(str(d['quotes']['USD']['price'])) break else: raise ValueError('malformed cryptocurrency data') for k in ('id', 'symbol'): - for d in data['cc']: + for d in data['cc'].data: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: @@ -425,7 +423,7 @@ def gen_data(data): def fi(): get_id = src_cls['fi'].get_id conv_func = src_cls['fi'].conv_data - for k, v in data['fi'].items(): + for k, v in data['fi'].data.items(): id = get_id(k, v) if wants['id']: if id in wants['id']: @@ -451,7 +449,7 @@ def gen_data(data): def hi(): ret = namedtuple('historical_closing_prices', ['close_1wk', 'close_4wks', 'close_1y']) nonlocal hist_close - for k, v in data['hi'].items(): + for k, v in data['hi'].data.items(): hist = tuple(v.values()) hist_close[k] = ret(hist[-2]['close'], hist[-5]['close'], hist[0]['close']) return () @@ -489,10 +487,15 @@ def gen_data(data): except Exception as e: rmsg(f'Error in source data {data_type!r}: {e}') parse_fail = True + else: + cache_data(data[data_type]) if parse_fail: die(2, 'Invalid data encountered, exiting') + if gcfg.download: + return + check_assets_found(usr_wants, found) for asset in (cfg.usr_rows + cfg.usr_columns): @@ -518,6 +521,18 @@ def gen_data(data): 'price_btc': Decimal(1) / btcusd, 'last_updated': None}) +def cache_data(data_src, no_overwrite=False): + if data_src.json_text: + if os.path.exists(data_src.json_fn): + if no_overwrite: + return False + os.rename(data_src.json_fn, data_src.json_fn + '.bak') + with open(data_src.json_fn, 'w') as fh: + fh.write(data_src.json_text) + if not cfg.quiet: + msg(f'JSON data cached to {data_src.json_fn_disp}') + return True + def main(): def update_sample_file(usr_cfg_file): @@ -558,7 +573,7 @@ def main(): return if gcfg.list_ids: - do_pager('\n'.join(e['id'] for e in src_data['cc'])) + do_pager('\n'.join(e.data['id'] for e in src_data['cc'])) return global now @@ -566,6 +581,9 @@ def main(): data = dict(gen_data(src_data)) + if gcfg.download: + return + (do_pager if cfg.pager else Msg_r)( '\n'.join(getattr(Ticker, cfg.clsname)(data).gen_output()) + '\n') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f9d163c..4e5cebc 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev3 +3.6.dev4 From 9ffaed6a912a4a42c84f5e9675a0d70d5edd9fae Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:58:54 +0000 Subject: [PATCH 090/108] mmnode-ticker: improve options setting Options set in the cfg file can now be unset on the command line, e.g.: $ mmnode-ticker --no-quiet --no-cached-data --- mmgen_node_tools/Ticker.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bbca8ef..49ce857 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -712,6 +712,12 @@ def make_cfg(gcfg_arg): rows += (hdr,) + uniq_data return rows + def get_cfg_var(name): + if name in gcfg._uopts: + return getattr(gcfg, name) + else: + return getattr(gcfg, name) or cfg_in.cfg.get(name) + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -772,27 +778,24 @@ def make_cfg(gcfg_arg): query = query, adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', - btc_only = gcfg.btc or cfg_in.cfg.get('btc'), - add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), - cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, + btc_only = get_cfg_var('btc'), + add_prec = parse_add_precision(get_cfg_var('add_precision')), + cachedir = get_cfg_var('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = - get_portfolio() - if cfg_in.portfolio - and (gcfg.portfolio or cfg_in.cfg.get('portfolio')) - and not query + get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query else None, - percent_cols = parse_percent_cols(gcfg.percent_cols or cfg_in.cfg.get('percent_cols')), - asset_limit = gcfg.asset_limit or cfg_in.cfg.get('asset_limit'), - cached_data = gcfg.cached_data or cfg_in.cfg.get('cached_data'), - elapsed = gcfg.elapsed or cfg_in.cfg.get('elapsed'), - name_labels = gcfg.name_labels or cfg_in.cfg.get('name_labels'), - pager = gcfg.pager or cfg_in.cfg.get('pager'), - thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), - update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), - quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), - verbose = gcfg.verbose or cfg_in.cfg.get('verbose')) + percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), + asset_limit = get_cfg_var('asset_limit'), + cached_data = get_cfg_var('cached_data'), + elapsed = get_cfg_var('elapsed'), + name_labels = get_cfg_var('name_labels'), + pager = get_cfg_var('pager'), + thousands_comma = get_cfg_var('thousands_comma'), + update_time = get_cfg_var('update_time'), + quiet = get_cfg_var('quiet'), + verbose = get_cfg_var('verbose')) def get_cfg_in(): ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) From 083b29eae829d24be0714355f759d1258300da87 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:59:00 +0000 Subject: [PATCH 091/108] mmnode-ticker: minor fixes and cleanups --- mmgen_node_tools/Ticker.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 49ce857..9d1c9df 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf +from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -184,7 +184,7 @@ class DataSource: dfl_asset_limit = 2000 def __init__(self): - self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) + self.asset_limit = int(cfg.asset_limit) if is_int(cfg.asset_limit) else self.dfl_asset_limit def rate_limit_errmsg(self, elapsed): rem = self.timeout - elapsed @@ -573,7 +573,7 @@ def main(): return if gcfg.list_ids: - do_pager('\n'.join(e.data['id'] for e in src_data['cc'])) + do_pager('\n'.join(e['id'] for e in src_data['cc'].data)) return global now @@ -606,7 +606,7 @@ def make_cfg(gcfg_arg): return tuple(gen()) def parse_percent_cols(arg): - if arg is None: + if arg is None or arg.lower() in ('none', ''): return [] res = arg.lower().split(',') for s in res: @@ -638,6 +638,8 @@ def make_cfg(gcfg_arg): source = parsed_id.source) cl_opt = getattr(gcfg, key) + if (cl_opt or '').lower() in ('', 'none'): + return () cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) @@ -718,6 +720,12 @@ def make_cfg(gcfg_arg): else: return getattr(gcfg, name) or cfg_in.cfg.get(name) + def get_proxy(name): + proxy = getattr(gcfg, name) + return ( + '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') + else (proxy or cfg_in.cfg.get(name))) + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -749,7 +757,6 @@ def make_cfg(gcfg_arg): src_cls = {k: getattr(DataSource, v) for k, v in DataSource.get_sources().items()} fi_pat = src_cls['fi'].asset_id_pat - cmd_args = gcfg._args cfg_in = get_cfg_in() if gcfg.test_suite: # required for testing with overlay @@ -757,15 +764,15 @@ def make_cfg(gcfg_arg): this_mod.src_cls = src_cls this_mod.cfg_in = cfg_in + if cmd_args := gcfg._args: + if len(cmd_args) > 1: + die(1, 'Only one command-line argument is allowed') + query = parse_query_arg(cmd_args[0]) + else: + query = None + usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) - query = parse_query_arg(cmd_args[0]) if cmd_args else None - - def get_proxy(name): - proxy = getattr(gcfg, name) - return ( - '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') - else (proxy or cfg_in.cfg.get(name))) proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy From 060b968ad4b88f845b9cd95fa89174a69b4b9743 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:59:00 +0000 Subject: [PATCH 092/108] mmnode-ticker: display crypto assets by market cap Examples: # Display top 2000 assets by market cap: $ mmnode-ticker 2000 # Display assets 201-300 by market cap, displaying all available columns: $ mmnode-ticker --widest 201-300 # Display asset 32 by market cap: $ mmnode-ticker 32-32 Testing/demo: $ test/cmdtest.py -e scripts.ticker --- mmgen_node_tools/Ticker.py | 53 +++++++++++++++++++++++++++++---- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 25 +++++++++++----- test/cmdtest_d/misc.py | 42 ++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9d1c9df..bf3398b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -182,6 +182,7 @@ class DataSource: net_data_type = 'json' has_verbose = True dfl_asset_limit = 2000 + max_asset_idx = 1_000_000 def __init__(self): self.asset_limit = int(cfg.asset_limit) if is_int(cfg.asset_limit) else self.dfl_asset_limit @@ -576,6 +577,17 @@ def main(): do_pager('\n'.join(e['id'] for e in src_data['cc'].data)) return + global cfg + + if cfg.asset_range: + func = DataSource.coinpaprika.parse_asset_id + n, m = cfg.asset_range + cfg = cfg._replace(rows = + tuple(func(e['id'], require_label=False) for e in src_data['cc'].data[n-1:m]) + + tuple(['-']) + + tuple([func('btc-bitcoin', require_label=True)]) + + tuple(r for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi')) + global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 @@ -643,6 +655,21 @@ def make_cfg(gcfg_arg): cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) + def parse_asset_range(s): + max_idx = DataSource.coinpaprika.max_asset_idx + match s.split('-'): + case [a, b] if is_int(a) and is_int(b): + n, m = (int(a), int(b)) + case [a] if is_int(a): + n, m = (1, int(a)) + case _: + return None + if n < 1 or m < 1 or n > m: + raise ValueError(f'‘{s}’: invalid asset range specifier') + if m > max_idx: + raise ValueError(f'‘{s}’: end of range must be <= {max_idx}') + return (n, m) + def parse_query_arg(s): """ asset_id:amount[:to_asset_id[:to_amount]] @@ -731,6 +758,7 @@ def make_cfg(gcfg_arg): 'usr_rows', 'usr_columns', 'query', + 'asset_range', 'adjust', 'clsname', 'btc_only', @@ -767,8 +795,10 @@ def make_cfg(gcfg_arg): if cmd_args := gcfg._args: if len(cmd_args) > 1: die(1, 'Only one command-line argument is allowed') - query = parse_query_arg(cmd_args[0]) + asset_range = parse_asset_range(cmd_args[0]) + query = None if asset_range else parse_query_arg(cmd_args[0]) else: + asset_range = None query = None usr_rows = parse_usr_asset_arg('add_rows') @@ -783,6 +813,7 @@ def make_cfg(gcfg_arg): usr_rows = usr_rows, usr_columns = usr_columns, query = query, + asset_range = asset_range, adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', btc_only = get_cfg_var('btc'), @@ -930,12 +961,14 @@ class Ticker: if self.table_hdr: yield self.table_hdr - for row in self.rows: + for n, row in enumerate(self.rows, cfg.asset_range[0] if cfg.asset_range else 1): if isinstance(row, str): + if cfg.asset_range: + return yield ('-' * self.hl_wid) else: try: - yield self.fmt_row(self.data[row.id]) + yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') @@ -989,7 +1022,7 @@ class Ticker: d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids} - def fmt_row(self, d, amt=None, amt_fmt=None): + def fmt_row(self, d, amt=None, amt_fmt=None, idx=None): def fmt_pct(n): return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') @@ -1002,6 +1035,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( + idx = idx, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), @@ -1061,6 +1095,15 @@ class Ticker: self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) + if cfg.asset_range: + def get_col1_w(): + for n, r in enumerate(cfg.rows): + if isinstance(r, str): + return len(str(n)) + col1_w = get_col1_w() + self.fs_str = ' ' * (col1_w + 2) + self.fs_str + self.fs_num = f'{{idx:{col1_w}}}) ' + self.fs_num + @property def table_hdr(self): return self.fs_str.format( @@ -1136,7 +1179,7 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self, d): + def fmt_row(self, d, idx=None): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4e5cebc..636c831 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev4 +3.6.dev5 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c36d886..8efb97a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -25,7 +25,7 @@ opts_data = { ], 'text': { 'desc': 'Display prices for cryptocurrency and other assets', - 'usage': '[opts] [TRADE_SPECIFIER]', + 'usage': '[opts] [TRADE_SPECIFIER | ASSET_RANGE]', 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) @@ -71,14 +71,19 @@ opts_data = { """, 'notes': """ -The script has two display modes: ‘overview’, the default, and ‘trading’, the -latter being enabled when a TRADE_SPECIFIER argument (see below) is supplied -on the command line. +The script has three display modes: ‘overview’, enabled when no arguments are +given on the command line; ‘trading’, when a TRADE_SPECIFIER argument (see +below) is given; and ‘market cap’, when an ASSET_RANGE (see below) is given. Overview mode displays prices of all configured assets, and optionally the -user’s portfolio, while trading mode displays the price of a given quantity -of an asset in relation to other assets, optionally comparing an offered -price to the spot price. +user’s portfolio; trading mode displays the price of a given quantity of an +asset in relation to other assets, optionally comparing an offered price to +the spot price; and market cap mode lists a range of crypto assets selected +by current market cap. + +The ASSET_RANGE argument can be either an integer N, in which case the top +N assets by market cap will be displayed, or a hyphen-separated range N-M, +in which case assets from N to M by market cap will be displayed. ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids) consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the @@ -199,6 +204,12 @@ $ mmnode-ticker usd:2700:btc:0.123 # current spot price, at specified USDINR rate: $ mmnode-ticker -n -c inr-indian-rupee:79.5 inr:200000:btc:0.1 +# Display top 20 crypto assets by market cap, adding a Euro column: +$ mmnode-ticker -c eurusd=x 20 + +# Same as above, specifying assets using a range: +$ mmnode-ticker -c eurusd=x 1-20 + CONFIGURED ASSETS: {assets} diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 1cde41e..c2dceff 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -86,6 +86,10 @@ class CmdTestScripts(CmdTestBase): ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), ('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'), + ('ticker18', 'ticker [--cached-data --widest --add-columns eurusd=x 10]'), + ('ticker19', 'ticker [--cached-data 1-5]'), + ('ticker20', 'ticker [--cached-data 2-5]'), + ('ticker21', 'ticker [--cached-data 5-5]'), ) } @@ -319,3 +323,41 @@ class CmdTestScripts(CmdTestBase): 'BITCOIN 23,250.77 42,731.767 1.00000000', 'BULGARIAN LEV 0.54 1.000 0.00002340', ]) + + def ticker18(self): + return self.ticker( + ['10'], + [ + r'1\) BITCOIN 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', + r'6\) ALGORAND 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + ], + add_opts = ['--widest', '--add-columns=eurusd=x']) + + def ticker19(self): + return self.ticker( + ['1-5'], + [ + 'USD EURUSD=X BTC ' + r'1\) BTC 23250.77 21848.7527 1.00000000', + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) + + def ticker20(self): + return self.ticker( + ['2-5'], + [ + 'USD EURUSD=X BTC ' + r'2\) ETH 1659.66 1559.5846 0.07138094', + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) + + def ticker21(self): + return self.ticker( + ['5-5'], + [ + 'USD EURUSD=X BTC ' + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) From c1f42fc25b123c334262e8cb53b460753e1c0be2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:14 +0000 Subject: [PATCH 093/108] mmnode-ticker: various fixes and cleanups --- mmgen_node_tools/Ticker.py | 135 +++++++++++++++++--------------- mmgen_node_tools/main_ticker.py | 4 +- test/cmdtest_d/misc.py | 6 +- 3 files changed, 81 insertions(+), 64 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bf3398b..7122e44 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -607,12 +607,10 @@ def make_cfg(gcfg_arg): def parse_asset_id(s, require_label=False): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) - def get_rows_from_cfg(add_data=None): + def get_rows_from_cfg(): def gen(): - for n, (k, v) in enumerate(cfg_in.cfg['assets'].items()): + for k, v in cfg_in.cfg['assets'].items(): yield k - if add_data and k in add_data: - v += tuple(add_data[k]) for e in v: yield parse_asset_id(e, require_label=True) return tuple(gen()) @@ -650,7 +648,7 @@ def make_cfg(gcfg_arg): source = parsed_id.source) cl_opt = getattr(gcfg, key) - if (cl_opt or '').lower() in ('', 'none'): + if cl_opt is None or cl_opt.lower() in ('none', ''): return () cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) @@ -702,15 +700,16 @@ def make_cfg(gcfg_arg): def get_usr_assets(): return ( - 'user_added', - usr_rows + - (tuple(asset for asset in query if asset) if query else ()) + - usr_columns) + usr_rows + + (tuple(asset for asset in query if asset) if query else ()) + + usr_columns) - def get_portfolio_assets(ret=()): + def get_portfolio_assets(): if cfg_in.portfolio and gcfg.portfolio: ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) - return ('portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC')) + return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') + else: + return () def get_portfolio(): return {k: Decimal(v) for k, v in cfg_in.portfolio.items() @@ -730,14 +729,12 @@ def make_cfg(gcfg_arg): rows = ( ('trade_pair',) + query if (query and query.to_asset) else ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg(add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None)) - + get_rows_from_cfg()) for hdr, data in ( - (get_usr_assets(),) if query else - (get_usr_assets(), get_portfolio_assets())): + ('user_uniq', get_usr_assets()), + ('portfolio_uniq', get_portfolio_assets())): if data: - uniq_data = tuple(gen_uniq(data, 'symbol', preload=rows)) - if uniq_data: + if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): rows += (hdr,) + uniq_data return rows @@ -787,11 +784,6 @@ def make_cfg(gcfg_arg): cfg_in = get_cfg_in() - if gcfg.test_suite: # required for testing with overlay - from . import Ticker as this_mod - this_mod.src_cls = src_cls - this_mod.cfg_in = cfg_in - if cmd_args := gcfg._args: if len(cmd_args) > 1: die(1, 'Only one command-line argument is allowed') @@ -808,6 +800,13 @@ def make_cfg(gcfg_arg): proxy = None if proxy == 'none' else proxy proxy2 = get_proxy('proxy2') + portfolio = ( + get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query + else None) + + if portfolio and asset_range: + die(1, '--portfolio not supported in market cap view') + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, @@ -821,9 +820,7 @@ def make_cfg(gcfg_arg): cachedir = get_cfg_var('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = - get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query - else None, + portfolio = portfolio, percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), @@ -835,6 +832,8 @@ def make_cfg(gcfg_arg): quiet = get_cfg_var('quiet'), verbose = get_cfg_var('verbose')) + return (src_cls, cfg_in) + def get_cfg_in(): ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) cfg_file, portfolio_file = ( @@ -871,7 +870,7 @@ class Ticker: self.col1_wid = max(len('TOTAL'), ( max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else - max(len(d['symbol']) for d in data.values()))) + 1 + max(len(d['symbol']) for d in data.values()))) self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row for row in cfg.rows] @@ -881,7 +880,7 @@ class Ticker: for row in self.rows if isinstance(row, tuple) and row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') - def format_last_update_col(self, cross_assets=()): + def format_last_updated_col(self, cross_assets=()): if cfg.elapsed: from mmgen.util2 import format_elapsed_hr @@ -899,19 +898,20 @@ class Ticker: min_t = None for row in self.rows: - if isinstance(row, tuple): - try: - t = int(d[row.id]['last_updated']) - except TypeError as e: - d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) - except KeyError as e: - msg(str(e)) - pass - else: - t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( - (min(t, min_t) if min_t else t), - now = now) - max_w = max(len(t_fmt), max_w) + if not isinstance(row, tuple): + continue + try: + t = int(d[row.id]['last_updated']) + except TypeError as e: + d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) + except KeyError as e: + msg(str(e)) + pass + else: + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( + (min(t, min_t) if min_t else t), + now = now) + max_w = max(len(t_fmt), max_w) self.upd_w = max_w @@ -924,8 +924,9 @@ class Ticker: if asset.id: return asset.id else: + m = asset.symbol for d in self.data.values(): - if d['symbol'] == asset.symbol: + if m == d['symbol']: return d['id'] def create_label(self, id): @@ -961,16 +962,26 @@ class Ticker: if self.table_hdr: yield self.table_hdr - for n, row in enumerate(self.rows, cfg.asset_range[0] if cfg.asset_range else 1): - if isinstance(row, str): - if cfg.asset_range: - return - yield ('-' * self.hl_wid) - else: + if cfg.asset_range: + yield '-' * self.hl_wid + for n, row in enumerate(self.rows, cfg.asset_range[0]): + if isinstance(row, str): + break try: yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') + else: + for row in self.rows: + if isinstance(row, str): + if cfg.asset_range: + return + yield ('-' * self.hl_wid) + else: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') yield '-' * self.hl_wid @@ -1003,7 +1014,7 @@ class Ticker: super().__init__(data) - self.format_last_update_col() + self.format_last_updated_col() if cfg.portfolio: self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] @@ -1072,9 +1083,9 @@ class Ticker: ) for k in self.col_ids} cols = ( - ['label', 'usd-us-dollar'] + - [asset.id for asset in self.usr_col_assets] + - [a for a, b in ( + ['label', 'usd-us-dollar'] + + [asset.id for asset in self.usr_col_assets] + + [a for a, b in ( ('btc-bitcoin', not cfg.btc_only), ('pct1y', 'y' in cfg.percent_cols), ('pct1m', 'm' in cfg.percent_cols), @@ -1082,6 +1093,17 @@ class Ticker: ('pct1d', 'd' in cfg.percent_cols), ('update_time', cfg.update_time)) if b]) + + if cfg.asset_range: + def get_num_w(): + for n, r in enumerate(cfg.rows): + if isinstance(r, str): + return len(str(n)) + num_w = get_num_w() + col_fs_data.update({ + 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) + cols = ['idx'] + cols + cols2 = list(cols) if cfg.update_time: cols2.pop() @@ -1095,15 +1117,6 @@ class Ticker: self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) - if cfg.asset_range: - def get_col1_w(): - for n, r in enumerate(cfg.rows): - if isinstance(r, str): - return len(str(n)) - col1_w = get_col1_w() - self.fs_str = ' ' * (col1_w + 2) + self.fs_str - self.fs_num = f'{{idx:{col1_w}}}) ' + self.fs_num - @property def table_hdr(self): return self.fs_str.format( @@ -1152,7 +1165,7 @@ class Ticker: for a in self.usr_col_assets: self.prices[a.id]['usd-us-dollar'] = data[a.id]['price_usd'] - self.format_last_update_col(cross_assets=self.usr_col_assets) + self.format_last_updated_col(cross_assets=self.usr_col_assets) self.init_prec() self.init_fs() diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 8efb97a..c6b41b3 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -244,9 +244,9 @@ from . import Ticker gcfg = Config(opts_data=opts_data, caller_post_init=True) -Ticker.make_cfg(gcfg) +src_cls, cfg_in = Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, cfg_in, src_cls +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen gcfg._post_init() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index c2dceff..b64e33c 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -338,8 +338,10 @@ class CmdTestScripts(CmdTestBase): ['1-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'1\) BTC 23250.77 21848.7527 1.00000000', - r'5\) ADA 0.51 0.4764 0.00002180', + r'5\) ADA 0.51 0.4764 0.00002180' + ' ---------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -348,6 +350,7 @@ class CmdTestScripts(CmdTestBase): ['2-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'2\) ETH 1659.66 1559.5846 0.07138094', r'5\) ADA 0.51 0.4764 0.00002180', ], @@ -358,6 +361,7 @@ class CmdTestScripts(CmdTestBase): ['5-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'5\) ADA 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From 3f921d333ccba378a6309123214e0a851e707ae8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 094/108] Ticker.py: `parse_asset_id()`: unify call signature --- mmgen_node_tools/Ticker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7122e44..76d9d2a 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -233,7 +233,7 @@ class DataSource: return [data] if cfg.btc_only else data @staticmethod - def parse_asset_id(s, require_label): + def parse_asset_id(s, require_label=True): sym, label = (*s.split('-', 1), None)[:2] if require_label and not label: die(1, f'{s!r}: asset label is missing') @@ -314,7 +314,7 @@ class DataSource: return ticker.price @staticmethod - def parse_asset_id(s, require_label): + def parse_asset_id(s, require_label=True): return asset_tuple( symbol = s.upper(), id = s.lower(), @@ -604,7 +604,7 @@ def make_cfg(gcfg_arg): query_tuple = namedtuple('query', ['asset', 'to_asset']) asset_data = namedtuple('asset_data', ['symbol', 'id', 'amount', 'rate', 'rate_asset', 'source']) - def parse_asset_id(s, require_label=False): + def parse_asset_id(s, require_label=True): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) def get_rows_from_cfg(): @@ -634,7 +634,7 @@ def make_cfg(gcfg_arg): ss = s.split(':') assert len(ss) in (1, 2, 3), f'{s}: malformed argument' asset_id, rate, rate_asset = (*ss, None, None)[:3] - parsed_id = parse_asset_id(asset_id) + parsed_id = parse_asset_id(asset_id, require_label=False) return asset_data( symbol = parsed_id.symbol, @@ -644,7 +644,7 @@ def make_cfg(gcfg_arg): None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else Decimal(rate)), - rate_asset = parse_asset_id(rate_asset) if rate_asset else None, + rate_asset = parse_asset_id(rate_asset, require_label=False) if rate_asset else None, source = parsed_id.source) cl_opt = getattr(gcfg, key) @@ -673,7 +673,7 @@ def make_cfg(gcfg_arg): asset_id:amount[:to_asset_id[:to_amount]] """ def parse_query_asset(asset_id, amount): - parsed_id = parse_asset_id(asset_id) + parsed_id = parse_asset_id(asset_id, require_label=False) return asset_data( symbol = parsed_id.symbol, id = parsed_id.id, @@ -706,7 +706,7 @@ def make_cfg(gcfg_arg): def get_portfolio_assets(): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) + ret = (parse_asset_id(e) for e in cfg_in.portfolio) return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') else: return () From 253aa14a265f66de3b97f826c0bef4d5ff3a369d Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 095/108] Ticker.py: new `RowDict` class --- mmgen_node_tools/Ticker.py | 86 +++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 48 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 76d9d2a..ed68eda 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -42,6 +42,11 @@ percent_cols = { 'y': 'year', } +class RowDict(dict): + + def __iter__(self): + return (e for v in self.values() for e in v) + class DataSource: source_groups = [ @@ -286,7 +291,7 @@ class DataSource: @property def symbols(self): - return [r.symbol for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi'] + return [r.symbol for r in cfg.rows if r.source == 'fi'] def get_data_from_network(self): @@ -456,8 +461,8 @@ def gen_data(data): return () rows_want = { - 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, - 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} + 'id': {r.id for r in cfg.rows if r.id} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if r.id is None} - {'USD'}} usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) usr_rate_assets_want = { 'id': {a.id for a in usr_rate_assets if a.id}, @@ -580,13 +585,16 @@ def main(): global cfg if cfg.asset_range: - func = DataSource.coinpaprika.parse_asset_id n, m = cfg.asset_range - cfg = cfg._replace(rows = - tuple(func(e['id'], require_label=False) for e in src_data['cc'].data[n-1:m]) - + tuple(['-']) - + tuple([func('btc-bitcoin', require_label=True)]) - + tuple(r for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi')) + cfg = cfg._replace(rows = RowDict({ + 'asset_list': + tuple( + asset_tuple(e['symbol'], e['id'], source='cc') + for e in src_data['cc'].data[n-1:m]), + 'extra': + tuple( + [asset_tuple('BTC', 'btc-bitcoin', source='cc')] + + [r for r in cfg.rows if r.source == 'fi'])})) global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 @@ -607,14 +615,6 @@ def make_cfg(gcfg_arg): def parse_asset_id(s, require_label=True): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) - def get_rows_from_cfg(): - def gen(): - for k, v in cfg_in.cfg['assets'].items(): - yield k - for e in v: - yield parse_asset_id(e, require_label=True) - return tuple(gen()) - def parse_percent_cols(arg): if arg is None or arg.lower() in ('none', ''): return [] @@ -726,16 +726,18 @@ def make_cfg(gcfg_arg): return int(s) def create_rows(): - rows = ( - ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg()) + rows = RowDict( + {'trade_pair': query} if (query and query.to_asset) else + {'bitcoin': [parse_asset_id('btc-bitcoin')]} if gcfg.btc else + {k: tuple(parse_asset_id(e) for e in v) for k, v in cfg_in.cfg['assets'].items()}) for hdr, data in ( ('user_uniq', get_usr_assets()), ('portfolio_uniq', get_portfolio_assets())): if data: if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): - rows += (hdr,) + uniq_data + rows[hdr] = uniq_data + else: + rows[hdr] = () return rows def get_cfg_var(name): @@ -872,12 +874,10 @@ class Ticker: max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else max(len(d['symbol']) for d in data.values()))) - self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row - for row in cfg.rows] + self.rows = RowDict( + {k: tuple(row._replace(id=self.get_id(row)) for row in v) for k, v in cfg.rows.items()}) self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} - - self.prices = {row.id: self.get_row_prices(row.id) - for row in self.rows if isinstance(row, tuple) and row.id in data} + self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') def format_last_updated_col(self, cross_assets=()): @@ -898,8 +898,6 @@ class Ticker: min_t = None for row in self.rows: - if not isinstance(row, tuple): - continue try: t = int(d[row.id]['last_updated']) except TypeError as e: @@ -964,24 +962,20 @@ class Ticker: if cfg.asset_range: yield '-' * self.hl_wid - for n, row in enumerate(self.rows, cfg.asset_range[0]): - if isinstance(row, str): - break + for n, row in enumerate(self.rows['asset_list'], cfg.asset_range[0]): try: yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') else: - for row in self.rows: - if isinstance(row, str): - if cfg.asset_range: - return - yield ('-' * self.hl_wid) - else: - try: - yield self.fmt_row(self.data[row.id]) - except KeyError: - yield gray(f'(no data for {row.id})') + for rows in self.rows.values(): + if rows: + yield '-' * self.hl_wid + for row in rows: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') yield '-' * self.hl_wid @@ -1019,7 +1013,7 @@ class Ticker: if cfg.portfolio: self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] for row in self.rows - if isinstance(row, tuple) and row.id in cfg.portfolio and row.id in data) + if row.id in cfg.portfolio and row.id in data) for col_id in self.col_ids} self.init_prec() @@ -1095,11 +1089,7 @@ class Ticker: if b]) if cfg.asset_range: - def get_num_w(): - for n, r in enumerate(cfg.rows): - if isinstance(r, str): - return len(str(n)) - num_w = get_num_w() + num_w = len(str(len(cfg.rows['asset_list']))) col_fs_data.update({ 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) cols = ['idx'] + cols From 9aa4b4dcfef0d95e96513042e42cc86c6bc997ba Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 096/108] mmnode-ticker: add `MarketCap` column Column is enabled automatically when script is invoked in market cap mode. --- mmgen_node_tools/Ticker.py | 8 ++++++-- mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 28 ++++++++++++++-------------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index ed68eda..9e598de 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -416,6 +416,7 @@ def gen_data(data): d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] + d['market_cap'] = d['quotes']['USD']['market_cap'] d['last_updated'] = int(datetime.datetime.fromisoformat( d['last_updated']).timestamp()) yield (d['id'], d) @@ -1041,6 +1042,7 @@ class Ticker: return self.fs_num.format( idx = idx, + mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), @@ -1091,8 +1093,9 @@ class Ticker: if cfg.asset_range: num_w = len(str(len(cfg.rows['asset_list']))) col_fs_data.update({ - 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) - cols = ['idx'] + cols + 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2), + 'mcap': fd('{mcap:>12}', '{mcap:12.5f}', 12)}) + cols = ['idx', 'label', 'mcap'] + cols[1:] cols2 = list(cols) if cfg.update_time: @@ -1111,6 +1114,7 @@ class Ticker: def table_hdr(self): return self.fs_str.format( lbl = '', + mcap = 'MarketCap(B)', pc1 = ' CHG_7d', pc2 = 'CHG_24h', pc3 = 'CHG_1y', diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 636c831..921f776 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev5 +3.6.dev6 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index b64e33c..a8d259e 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -328,8 +328,8 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['10'], [ - r'1\) BITCOIN 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', - r'6\) ALGORAND 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + r'1\) BITCOIN 444.33652 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', + r'6\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' ], add_opts = ['--widest', '--add-columns=eurusd=x']) @@ -337,11 +337,11 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['1-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'1\) BTC 23250.77 21848.7527 1.00000000', - r'5\) ADA 0.51 0.4764 0.00002180' - ' ---------------------------------------------' + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'1\) BTC 444.33652 23250.77 21848.7527 1.00000000', + r'5\) ADA 17.11161 0.51 0.4764 0.00002180' + ' ---------------------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -349,10 +349,10 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['2-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'2\) ETH 1659.66 1559.5846 0.07138094', - r'5\) ADA 0.51 0.4764 0.00002180', + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'2\) ETH 202.15129 1659.66 1559.5846 0.07138094', + r'5\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) @@ -360,8 +360,8 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['5-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'5\) ADA 0.51 0.4764 0.00002180', + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'5\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From e71ef141bffd8f44998e1510edf197f423cb2e3b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:09 +0000 Subject: [PATCH 097/108] mmnode-ticker: minor cleanups --- mmgen_node_tools/Ticker.py | 20 ++++++++++---------- mmgen_node_tools/main_ticker.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9e598de..6aef36f 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red, yellow, green, blue, orange, gray +from mmgen.color import red, yellow, green, blue, orange, gray, cyan from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager @@ -39,8 +39,7 @@ percent_cols = { 'd': 'day', 'w': 'week', 'm': 'month', - 'y': 'year', -} + 'y': 'year'} class RowDict(dict): @@ -713,8 +712,8 @@ def make_cfg(gcfg_arg): return () def get_portfolio(): - return {k: Decimal(v) for k, v in cfg_in.portfolio.items() - if (not gcfg.btc) or k == 'btc-bitcoin'} + return tuple((k, Decimal(v)) for k, v in cfg_in.portfolio.items() + if (not gcfg.btc) or k == 'btc-bitcoin') def parse_add_precision(arg): if not arg: @@ -932,7 +931,7 @@ class Ticker: return self.data[id]['name'].upper() def gen_output(self): - yield 'Current time: {} UTC'.format(time.strftime('%F %X', time.gmtime(now))) + yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) for asset in self.usr_col_assets: if asset.symbol != 'USD': @@ -987,7 +986,7 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym, amt in cfg.portfolio.items(): + for sym, amt in cfg.portfolio: try: yield self.fmt_row(self.data[sym], amt=amt) except KeyError: @@ -1005,16 +1004,17 @@ class Ticker: self.adjust = cfg.adjust self.show_adj = self.adjust != 1 self.usr_col_assets = [asset._replace(id=self.get_id(asset)) for asset in cfg.usr_columns] - self.col_ids = ('usd-us-dollar',) + tuple(a.id for a in self.usr_col_assets) + ('btc-bitcoin',) + self.col_ids = ('usd-us-dollar', 'btc-bitcoin') + tuple(a.id for a in self.usr_col_assets) super().__init__(data) self.format_last_updated_col() if cfg.portfolio: - self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + pf_dict = dict(cfg.portfolio) + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * pf_dict[row.id] for row in self.rows - if row.id in cfg.portfolio and row.id in data) + if row.id in pf_dict and row.id in data) for col_id in self.col_ids} self.init_prec() diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c6b41b3..e533e62 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -225,7 +225,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, - pc = fmt_list(Ticker.percent_cols, fmt='bare')), + pc = fmt_list(Ticker.percent_cols, fmt='fancy')), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), cfg = os.path.relpath(cfg_in.cfg_file, start=homedir), From e6d62fd18b2d9e5885ce4452c843850871550f60 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:15 +0000 Subject: [PATCH 098/108] mmnode-ticker: display coin ranking in first column --- mmgen_node_tools/Ticker.py | 40 +++++++++++++++++++++++--------------- test/cmdtest_d/misc.py | 14 ++++++------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6aef36f..a8b6595 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -865,6 +865,7 @@ class Ticker: offer = None to_asset = None + hidden_groups = ('extra',) def __init__(self, data): @@ -876,6 +877,14 @@ class Ticker: self.rows = RowDict( {k: tuple(row._replace(id=self.get_id(row)) for row in v) for k, v in cfg.rows.items()}) + + if cfg.asset_range: + self.max_rank = 0 + for group, rows in self.rows.items(): + if group not in self.hidden_groups: + for row in rows: + self.max_rank = max(self.max_rank, int(data[row.id]['rank'])) + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -931,6 +940,15 @@ class Ticker: return self.data[id]['name'].upper() def gen_output(self): + + def process_rows(rows): + yield '-' * self.hl_wid + for row in rows: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') + yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) for asset in self.usr_col_assets: @@ -961,21 +979,11 @@ class Ticker: yield self.table_hdr if cfg.asset_range: - yield '-' * self.hl_wid - for n, row in enumerate(self.rows['asset_list'], cfg.asset_range[0]): - try: - yield self.fmt_row(self.data[row.id], idx=n) - except KeyError: - yield gray(f'(no data for {row.id})') + yield from process_rows(self.rows['asset_list']) else: for rows in self.rows.values(): if rows: - yield '-' * self.hl_wid - for row in rows: - try: - yield self.fmt_row(self.data[row.id]) - except KeyError: - yield gray(f'(no data for {row.id})') + yield from process_rows(rows) yield '-' * self.hl_wid @@ -1028,7 +1036,7 @@ class Ticker: d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids} - def fmt_row(self, d, amt=None, amt_fmt=None, idx=None): + def fmt_row(self, d, amt=None, amt_fmt=None): def fmt_pct(n): return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') @@ -1041,7 +1049,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( - idx = idx, + idx = int(d['rank']) if cfg.asset_range else None, mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), @@ -1091,7 +1099,7 @@ class Ticker: if b]) if cfg.asset_range: - num_w = len(str(len(cfg.rows['asset_list']))) + num_w = len(str(self.max_rank)) col_fs_data.update({ 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2), 'mcap': fd('{mcap:>12}', '{mcap:12.5f}', 12)}) @@ -1186,7 +1194,7 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self, d, idx=None): + def fmt_row(self, d): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index a8d259e..3e909f4 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -329,7 +329,7 @@ class CmdTestScripts(CmdTestBase): ['10'], [ r'1\) BITCOIN 444.33652 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', - r'6\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + r'33\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' ], add_opts = ['--widest', '--add-columns=eurusd=x']) @@ -338,10 +338,10 @@ class CmdTestScripts(CmdTestBase): ['1-5'], [ r'MarketCap\(B\) USD EURUSD=X BTC ' - '--------------------------------------------------------- ' + '---------------------------------------------------------- ' r'1\) BTC 444.33652 23250.77 21848.7527 1.00000000', - r'5\) ADA 17.11161 0.51 0.4764 0.00002180' - ' ---------------------------------------------------------' + r'8\) ADA 17.11161 0.51 0.4764 0.00002180' + ' ----------------------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -350,9 +350,9 @@ class CmdTestScripts(CmdTestBase): ['2-5'], [ r'MarketCap\(B\) USD EURUSD=X BTC ' - '--------------------------------------------------------- ' + '---------------------------------------------------------- ' r'2\) ETH 202.15129 1659.66 1559.5846 0.07138094', - r'5\) ADA 17.11161 0.51 0.4764 0.00002180', + r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) @@ -362,6 +362,6 @@ class CmdTestScripts(CmdTestBase): [ r'MarketCap\(B\) USD EURUSD=X BTC ' '--------------------------------------------------------- ' - r'5\) ADA 17.11161 0.51 0.4764 0.00002180', + r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From 0a953e3ca039040b9ca4dbff501913bc39b760bb Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:15 +0000 Subject: [PATCH 099/108] mmnode-ticker: sort output by various parameters Supported parameters: d - 1-day % change w - 1-week % change m - 1-month % change y - 1-year % change p - asset price c - market cap Examples: # Display top 50 assets by market cap, sorting by price change # in last 24 hours: $ mmnode-ticker --sort=d 50 --- mmgen_node_tools/Ticker.py | 45 ++++++++++++++++++++++- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 11 +++++- test/cmdtest_d/misc.py | 63 +++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index a8b6595..d3045d4 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red, yellow, green, blue, orange, gray, cyan +from mmgen.color import red, yellow, green, blue, orange, gray, cyan, pink from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager @@ -41,6 +41,15 @@ percent_cols = { 'm': 'month', 'y': 'year'} +sp = namedtuple('sort_parameter', ['key', 'desc']) +sort_params = { + 'd': sp('percent_change_24h', '1-day % change'), + 'w': sp('percent_change_7d', '1-week % change'), + 'm': sp('percent_change_30d', '1-month % change'), + 'y': sp('percent_change_1y', '1-year % change'), + 'p': sp('price_usd', 'asset price'), + 'c': sp('market_cap', 'market cap')} + class RowDict(dict): def __iter__(self): @@ -274,6 +283,7 @@ class DataSource: 'percent_change_30d': data['pct_chg_4wks'], 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, + 'market_cap': 0, # dummy - required for sorting 'last_updated': data['regularMarketTime']} def rate_limit_errmsg(self, elapsed): @@ -752,6 +762,19 @@ def make_cfg(gcfg_arg): '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') else (proxy or cfg_in.cfg.get(name))) + def get_sort_opt(): + match get_cfg_var('sort'): + case None: + return None + case s if s in sort_params: + return (s, True) + case s if s in ['r' + ch for ch in sort_params]: + return (s[1], False) + case s: + die(1, + f'{s!r}: invalid parameter for --sort option (must be one of {fmt_list(sort_params)})' + '\nTo reverse the sort, prefix the code letter with ‘r’') + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -766,6 +789,7 @@ def make_cfg(gcfg_arg): 'proxy', 'proxy2', 'portfolio', + 'sort', 'percent_cols', 'asset_limit', 'cached_data', @@ -823,6 +847,7 @@ def make_cfg(gcfg_arg): proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = portfolio, + sort = get_sort_opt(), percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), @@ -869,6 +894,8 @@ class Ticker: def __init__(self, data): + global cfg + self.comma = ',' if cfg.thousands_comma else '' self.col1_wid = max(len('TOTAL'), ( @@ -885,6 +912,18 @@ class Ticker: for row in rows: self.max_rank = max(self.max_rank, int(data[row.id]['rank'])) + if cfg.sort: + code, reverse = cfg.sort + key = sort_params[code].key + sort_func = lambda row: data[row.id][key] + pf_sort_func = lambda row: data[row[0]][key] + for group in self.rows.keys(): + if group not in self.hidden_groups: + self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) + if cfg.portfolio: + cfg = cfg._replace( + portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -951,6 +990,10 @@ class Ticker: yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) + if cfg.sort: + text = sort_params[cfg.sort[0]].desc + ('' if cfg.sort[1] else ' [reversed]') + yield f'Sort order: {pink(text.upper())}' + for asset in self.usr_col_assets: if asset.symbol != 'USD': usdprice = self.data[asset.id]['price_usd'] diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 921f776..1de14a5 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev6 +3.6.dev7 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index e533e62..7be5e90 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -57,6 +57,9 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. +-s, --sort=P Sort output according to parameter P. Valid parameters + are {sp_codes}. See SORT PARAMETERS below. + To reverse the sort, prefix the parameter with ‘r’. -t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column @@ -133,6 +136,10 @@ A TRADE_SPECIFIER is a single argument in the format: a USD rate for the missing asset(s) must be supplied via the --add-columns or --add-rows options. +SORT PARAMETERS: + + {sp_fmt} + PROXY NOTE @@ -225,6 +232,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, + sp_codes = fmt_list(sort_params, fmt='fancy'), pc = fmt_list(Ticker.percent_cols, fmt='fancy')), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), @@ -232,6 +240,7 @@ To add a portfolio, edit the file pf_cfg = os.path.relpath(cfg_in.portfolio_file, start=homedir), al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), + sp_fmt = '\n '.join(f'‘{k}’ - {v.desc}' for k, v in sort_params.items()), fi = src_cls['fi']()) } } @@ -246,7 +255,7 @@ gcfg = Config(opts_data=opts_data, caller_post_init=True) src_cls, cfg_in = Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, sort_params gcfg._post_init() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 3e909f4..275117f 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -90,6 +90,14 @@ class CmdTestScripts(CmdTestBase): ('ticker19', 'ticker [--cached-data 1-5]'), ('ticker20', 'ticker [--cached-data 2-5]'), ('ticker21', 'ticker [--cached-data 5-5]'), + ('ticker22', 'ticker [--sort=rp]'), + ('ticker23', 'ticker [--sort=rp xmr:10]'), + ('ticker24', 'ticker [--sort=p]'), + ('ticker25', 'ticker [--sort=p 200]'), + ('ticker26', 'ticker [--sort=c -r algo,ada]'), + ('ticker27', 'ticker [--sort=rp -r algo,ada]'), + ('ticker28', 'ticker [--sort=d -r algo,ada]'), + ('ticker29', 'ticker [--sort=y -r algo,ada]'), ) } @@ -365,3 +373,58 @@ class CmdTestScripts(CmdTestBase): r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) + + def ticker22(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], + add_opts = ['--name-labels', '--sort=rp']) + + def ticker23(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], + add_opts = ['--name-labels', '--sort=rp', 'xmr:10']) + + def ticker24(self): + return self.ticker( + [], + ['BITCOIN', 'ETHEREUM', 'MONERO', 'GOLD', 'BRENT', 'SILVER'], + add_opts = ['--name-labels', '--sort=p']) + + def ticker25(self): + return self.ticker( + [], + [ + r' 1\) BITCOIN', + r' 2\) ETHEREUM', + r'30\) MONERO', + r'23\) LITECOIN', + r' 8\) CARDANO', + r'33\) ALGORAND' + ], + add_opts = ['--name-labels', '--sort=p', '200']) + + def ticker26(self): + return self.ticker( + [], + ['BITCOIN', 'ETHEREUM', 'MONERO', 'CARDANO', 'ALGORAND'], + add_opts = ['--name-labels', '--sort=c', '-r', 'ada,algo']) + + def ticker27(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'S&P', 'NASDAQ', 'DOW', 'ALGORAND', 'CARDANO'], + add_opts = ['--name-labels', '--sort=rp', '--add-rows=ada-cardano,algo-algorand']) + + def ticker28(self): + return self.ticker( + [], + ['ETHEREUM', 'MONERO', 'BITCOIN', 'NASDAQ', 'S&P', 'DOW', 'CARDANO', 'ALGORAND'], + add_opts = ['--widest', '--sort=d', '-r', 'ada,algo']) + + def ticker29(self): + return self.ticker( + [], + ['ETHEREUM', 'BITCOIN', 'MONERO', 'S&P', 'DOW', 'NASDAQ', 'CARDANO', 'ALGORAND'], + add_opts = ['--widest', '-s', 'y', '-r', 'ada,algo']) From 13234e990bc70b2cd53b78faaed121fc584f8cd8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 17 Oct 2025 09:07:55 +0000 Subject: [PATCH 100/108] cmdtest.py misc.ticker: cleanups --- test/cmdtest_d/misc.py | 43 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 275117f..9060731 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -67,7 +67,7 @@ class CmdTestScripts(CmdTestBase): 'ticker': ( "'mmnode-ticker' script", ('ticker1', 'ticker [--help]'), - ('copy_files', 'copying JSON files to cache'), + ('copy_cache_files', 'copying JSON files to cache'), ('ticker1a', 'ticker [--download=cc] (early caching)'), ('ticker1b', 'ticker [--download=cc] (late caching)'), ('ticker2', 'ticker (bad proxy)'), @@ -106,17 +106,24 @@ class CmdTestScripts(CmdTestBase): return self.ticker_server = TickerServer(cfg) self.ticker_server.start() + self.dests = { + 'nt_datadir': os.path.join(cfg.data_dir_root, 'node_tools'), + 'cache': self.tmpdir} return super().__init__(cfg, trunner, cfgs, spawn) - @property - def nt_datadir(self): - return os.path.join( cfg.data_dir_root, 'node_tools' ) + def rm_file(self, fn, dest='nt_datadir'): + os.unlink(os.path.join(self.dests[dest], fn)) - def copy_files(self): - self.spawn('',msg_only=True) - shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) - shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) - shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) + def copy_file(self, src_fn, dest_fn=None, dest='nt_datadir'): + shutil.copy2( + os.path.join(refdir, src_fn), + os.path.join(self.dests[dest], dest_fn or src_fn)) + + def copy_cache_files(self): + self.spawn('', msg_only=True) + self.copy_file('ticker-finance.json', dest='cache') + self.copy_file('ticker-finance-history.json', dest='cache') + self.copy_file('ticker-btc.json', dest='cache') return 'ok' def ticker( @@ -186,7 +193,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker5(self): - shutil.copy2(os.path.join(refdir,'ticker-cfg.yaml'),self.nt_datadir) + self.copy_file('ticker-cfg.yaml') t = self.ticker( ['--wide','--adjust=-0.766'], [ @@ -195,7 +202,7 @@ class CmdTestScripts(CmdTestBase): r'LITECOIN 58.56 0.00251869 \+12.79 \+0.40 2022-08-02 18:25:59', r'MONERO 157.76 0.00678495 \+7.28 \+1.21' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-cfg.yaml')) + self.rm_file('ticker-cfg.yaml') return t def ticker6(self): @@ -204,7 +211,7 @@ class CmdTestScripts(CmdTestBase): return t def ticker7(self): # demo - shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + self.copy_file('ticker-portfolio.yaml') t = self.ticker( ['--wide','--portfolio'], [ @@ -213,7 +220,7 @@ class CmdTestScripts(CmdTestBase): 'CARDANO','ALGORAND', 'PORTFOLIO','BITCOIN','ETHEREUM','MONERO','CARDANO','ALGORAND','TOTAL' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker8(self): @@ -225,9 +232,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker9(self): - shutil.copy2( - os.path.join(refdir,'ticker-portfolio-bad.yaml'), - os.path.join(self.nt_datadir,'ticker-portfolio.yaml') ) + self.copy_file('ticker-portfolio-bad.yaml', 'ticker-portfolio.yaml') t = self.ticker( ['--wide','--portfolio','--elapsed','--add-rows=fake-fakecoin:0.0123','--add-precision=2'], [ @@ -236,7 +241,7 @@ class CmdTestScripts(CmdTestBase): r'FAKECOIN 81.3008 0.0034966927 -- -- --', r'\(no data for noc-nocoin\)', ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker10(self): @@ -287,7 +292,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker14(self): - shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + self.copy_file('ticker-portfolio.yaml') t = self.ticker( ['--btc','--wide','--portfolio','--elapsed'], [ @@ -296,7 +301,7 @@ class CmdTestScripts(CmdTestBase): 'PORTFOLIO', r'BITCOIN 28,850.44 \+6.05 -1.87 1.23456789' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker15(self): From 8f9c4ba48ccf695b48d718c6d95a8b717ef85a4c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 17 Oct 2025 09:08:00 +0000 Subject: [PATCH 101/108] mmnode-ticker --sort: handle missing entries in source data --- mmgen_node_tools/Ticker.py | 19 ++++++++++--------- mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 5 ++++- test/ref/ticker/ticker-cfg-bad.yaml | 11 +++++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-bad.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index d3045d4..925a4f9 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -41,14 +41,14 @@ percent_cols = { 'm': 'month', 'y': 'year'} -sp = namedtuple('sort_parameter', ['key', 'desc']) +sp = namedtuple('sort_parameter', ['key', 'sort_dfl', 'desc']) sort_params = { - 'd': sp('percent_change_24h', '1-day % change'), - 'w': sp('percent_change_7d', '1-week % change'), - 'm': sp('percent_change_30d', '1-month % change'), - 'y': sp('percent_change_1y', '1-year % change'), - 'p': sp('price_usd', 'asset price'), - 'c': sp('market_cap', 'market cap')} + 'd': sp('percent_change_24h', 0.0, '1-day % change'), + 'w': sp('percent_change_7d', 0.0, '1-week % change'), + 'm': sp('percent_change_30d', 0.0, '1-month % change'), + 'y': sp('percent_change_1y', 0.0, '1-year % change'), + 'p': sp('price_usd', Decimal(0), 'asset price'), + 'c': sp('market_cap', 0, 'market cap')} class RowDict(dict): @@ -915,8 +915,9 @@ class Ticker: if cfg.sort: code, reverse = cfg.sort key = sort_params[code].key - sort_func = lambda row: data[row.id][key] - pf_sort_func = lambda row: data[row[0]][key] + sort_dfl = sort_params[code].sort_dfl + sort_func = lambda row: data.get(row.id, {key: sort_dfl})[key] + pf_sort_func = lambda row: data.get(row[0], {key: sort_dfl})[key] for group in self.rows.keys(): if group not in self.hidden_groups: self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1de14a5..6eb528d 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev7 +3.6.dev8 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9060731..a3edd34 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -380,10 +380,13 @@ class CmdTestScripts(CmdTestBase): add_opts = ['--add-columns=eurusd=x']) def ticker22(self): - return self.ticker( + self.copy_file('ticker-cfg-bad.yaml', 'ticker-cfg.yaml') + t = self.ticker( [], ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], add_opts = ['--name-labels', '--sort=rp']) + self.rm_file('ticker-cfg.yaml') + return t def ticker23(self): return self.ticker( diff --git a/test/ref/ticker/ticker-cfg-bad.yaml b/test/ref/ticker/ticker-cfg-bad.yaml new file mode 100644 index 0000000..da670c6 --- /dev/null +++ b/test/ref/ticker/ticker-cfg-bad.yaml @@ -0,0 +1,11 @@ +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + - bad-badcoin + commodity: + - gc=f + - si=f + - bz=f From 2647fa1fe3d511bec6ca841101c8e1472f479551 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Oct 2025 09:14:30 +0000 Subject: [PATCH 102/108] mmnode-ticker: minor cleanups --- mmgen_node_tools/Ticker.py | 30 ++++++++++++++++-------------- mmgen_node_tools/main_ticker.py | 4 ++-- test/cmdtest_d/misc.py | 2 ++ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 925a4f9..7bcb567 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -43,10 +43,10 @@ percent_cols = { sp = namedtuple('sort_parameter', ['key', 'sort_dfl', 'desc']) sort_params = { - 'd': sp('percent_change_24h', 0.0, '1-day % change'), - 'w': sp('percent_change_7d', 0.0, '1-week % change'), - 'm': sp('percent_change_30d', 0.0, '1-month % change'), - 'y': sp('percent_change_1y', 0.0, '1-year % change'), + 'd': sp('percent_change_24h', 0.0, '1-day percent change'), + 'w': sp('percent_change_7d', 0.0, '1-week percent change'), + 'm': sp('percent_change_30d', 0.0, '1-month percent change'), + 'y': sp('percent_change_1y', 0.0, '1-year percent change'), 'p': sp('price_usd', Decimal(0), 'asset price'), 'c': sp('market_cap', 0, 'market cap')} @@ -1025,8 +1025,8 @@ class Ticker: if cfg.asset_range: yield from process_rows(self.rows['asset_list']) else: - for rows in self.rows.values(): - if rows: + for group, rows in self.rows.items(): + if rows and group not in self.hidden_groups: yield from process_rows(rows) yield '-' * self.hl_wid @@ -1082,8 +1082,10 @@ class Ticker: def fmt_row(self, d, amt=None, amt_fmt=None): - def fmt_pct(n): - return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') + def fmt_pct(d, key, wid=7): + if (n := d.get(key)) is None: + return gray(' --') + return (red, green)[n>=0](f'{n:+{wid}.2f}') p = self.prices[d['id']] @@ -1096,10 +1098,10 @@ class Ticker: idx = int(d['rank']) if cfg.asset_range else None, mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], - pc1 = fmt_pct(d.get('percent_change_7d')), - pc2 = fmt_pct(d.get('percent_change_24h')), - pc3 = fmt_pct(d.get('percent_change_1y')), - pc4 = fmt_pct(d.get('percent_change_30d')), + pc1 = fmt_pct(d, 'percent_change_7d'), + pc2 = fmt_pct(d, 'percent_change_24h'), + pc3 = fmt_pct(d, 'percent_change_1y', wid=8), + pc4 = fmt_pct(d, 'percent_change_30d'), upd = d.get('last_updated_fmt'), amt = amt_fmt, **{k.replace('-', '_'): v * (1 if amt is None else amt) for k, v in p.items()}) @@ -1117,7 +1119,7 @@ class Ticker: col_fs_data = { 'label': fd(f'{{lbl:{self.col1_wid}}}', f'{{lbl:{self.col1_wid}}}', self.col1_wid), - 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), + 'pct1y': fd(' {pc3:8}', ' {pc3:8}', 9), 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), @@ -1169,7 +1171,7 @@ class Ticker: mcap = 'MarketCap(B)', pc1 = ' CHG_7d', pc2 = 'CHG_24h', - pc3 = 'CHG_1y', + pc3 = ' CHG_1y', pc4 = 'CHG_30d', upd = 'UPDATED', amt = ' AMOUNT', diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 7be5e90..d4c0055 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -40,10 +40,10 @@ opts_data = { used to supply a USD exchange rate for missing assets. -C, --cached-data Use cached data from previous network query instead of live data from server --D, --cachedir=D Read and write cached JSON data to directory ‘D’ - instead of ‘~/{dfl_cachedir}’ -d, --download=D Retrieve and cache asset data ‘D’ from network (valid options: {ds}) +-D, --cachedir=D Read and write cached JSON data to directory ‘D’ + instead of ‘~/{dfl_cachedir}’ -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index a3edd34..c2a4188 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -133,6 +133,7 @@ class CmdTestScripts(CmdTestBase): cached_data = True, add_opts = [], use_proxy = True, + no_msg = False, exit_val = None): t = self.spawn( 'mmnode-ticker', @@ -141,6 +142,7 @@ class CmdTestScripts(CmdTestBase): + (['--proxy=http://asdfzxcv:32459'] if use_proxy else []) + add_opts + args, + no_msg = no_msg, exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) From 84e8ea65d0b86cab2896125e1f1022d4848742a8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Oct 2025 09:14:34 +0000 Subject: [PATCH 103/108] mmnode-ticker: display percent change columns in terms of non-USD assets Any crypto or finance asset may be specified. Examples: # Display percentage changes in relation to Bitcoin: $ mmnode-ticker --widest --pchg-unit=btc # In relation to Gold: $ mmnode-ticker --widest --pchg-unit=gc=f # In relation to Euros: $ mmnode-ticker --widest --pchg-unit=eurusd=x # In relation to the Nasdaq Index: $ mmnode-ticker --widest --pchg-unit=^ixic --- mmgen_node_tools/Ticker.py | 29 +++++++++++++++-- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 2 ++ test/cmdtest_d/misc.py | 38 +++++++++++++++++++++++ test/ref/ticker/ticker-cfg-sort-pchg.yaml | 14 +++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-sort-pchg.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7bcb567..4b49b44 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -535,6 +535,10 @@ def gen_data(data): 'name': 'US Dollar', 'price_usd': Decimal(1), 'price_btc': Decimal(1) / btcusd, + 'percent_change_24h': 0.0, + 'percent_change_7d': 0.0, + 'percent_change_30d': 0.0, + 'percent_change_1y': 0.0, 'last_updated': None}) def cache_data(data_src, no_overwrite=False): @@ -742,7 +746,8 @@ def make_cfg(gcfg_arg): {k: tuple(parse_asset_id(e) for e in v) for k, v in cfg_in.cfg['assets'].items()}) for hdr, data in ( ('user_uniq', get_usr_assets()), - ('portfolio_uniq', get_portfolio_assets())): + ('portfolio_uniq', get_portfolio_assets()), + ('pchg_unit_uniq', [pchg_unit] if pchg_unit else None)): if data: if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): rows[hdr] = uniq_data @@ -791,6 +796,7 @@ def make_cfg(gcfg_arg): 'portfolio', 'sort', 'percent_cols', + 'pchg_unit', 'asset_limit', 'cached_data', 'elapsed', @@ -833,6 +839,9 @@ def make_cfg(gcfg_arg): if portfolio and asset_range: die(1, '--portfolio not supported in market cap view') + pchg_unit = (lambda s: parse_asset_id(s, require_label=False) if s else None)( + get_cfg_var('pchg_unit')) + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, @@ -849,6 +858,7 @@ def make_cfg(gcfg_arg): portfolio = portfolio, sort = get_sort_opt(), percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), + pchg_unit = pchg_unit, asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), elapsed = get_cfg_var('elapsed'), @@ -890,7 +900,7 @@ class Ticker: offer = None to_asset = None - hidden_groups = ('extra',) + hidden_groups = ('extra', 'pchg_unit_uniq') def __init__(self, data): @@ -925,6 +935,14 @@ class Ticker: cfg = cfg._replace( portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + if cfg.pchg_unit: + self.pchg_data = self.data[self.get_id(cfg.pchg_unit)] + self.pchg_factors = {k: (self.pchg_data[k] / 100) + 1 for k in ( + 'percent_change_24h', + 'percent_change_7d', + 'percent_change_30d', + 'percent_change_1y')} + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -995,6 +1013,11 @@ class Ticker: text = sort_params[cfg.sort[0]].desc + ('' if cfg.sort[1] else ' [reversed]') yield f'Sort order: {pink(text.upper())}' + if cfg.pchg_unit: + yield 'Percent change unit: {}'.format(orange('{} ({})'.format( + self.pchg_data['symbol'], + self.pchg_data['name'].upper()))) + for asset in self.usr_col_assets: if asset.symbol != 'USD': usdprice = self.data[asset.id]['price_usd'] @@ -1085,6 +1108,8 @@ class Ticker: def fmt_pct(d, key, wid=7): if (n := d.get(key)) is None: return gray(' --') + if cfg.pchg_unit: + n = ((((n / 100) + 1) / self.pchg_factors[key]) - 1) * 100 return (red, green)[n>=0](f'{n:+{wid}.2f}') p = self.prices[d['id']] diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6eb528d..47203f7 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev8 +3.6.dev9 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index d4c0055..cfca2ac 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -63,6 +63,8 @@ opts_data = { -t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column +-U, --pchg-unit=A Use asset ‘A’ as unit of reference for percentage + change columns (default: USD) -v, --verbose Be more verbose -w, --wide Display most optional columns (same as -unT -p d,w) -W, --widest Display all optional columns (same as -unT -p d,w,m,y) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index c2a4188..9112f48 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -98,6 +98,9 @@ class CmdTestScripts(CmdTestBase): ('ticker27', 'ticker [--sort=rp -r algo,ada]'), ('ticker28', 'ticker [--sort=d -r algo,ada]'), ('ticker29', 'ticker [--sort=y -r algo,ada]'), + ('ticker30', 'ticker [--cached-data --wide --pchg-unit=btc --sort=d] (cf with config file)'), + ('ticker31', 'ticker [--cached-data --wide --pchg-unit=usd] (cf with no USD)'), + ('ticker32', 'ticker [--cached-data --wide --pchg-unit=gc=f]'), ) } @@ -438,3 +441,38 @@ class CmdTestScripts(CmdTestBase): [], ['ETHEREUM', 'BITCOIN', 'MONERO', 'S&P', 'DOW', 'NASDAQ', 'CARDANO', 'ALGORAND'], add_opts = ['--widest', '-s', 'y', '-r', 'ada,algo']) + + def ticker30(self): + self.copy_file('ticker-cfg-sort-pchg.yaml', 'ticker-cfg.yaml') + t = self.ticker(add_opts=['--wide']) + chk1 = '\n'.join(t.read().splitlines()[5:-2]) + self.rm_file('ticker-cfg.yaml') + + self.copy_file('ticker-cfg-bad.yaml', 'ticker-cfg.yaml') + t = self.ticker(add_opts=['--wide', '--pchg-unit=btc', '--sort=d'], no_msg=True) + chk2 = '\n'.join(t.read().splitlines()[5:-2]) + self.rm_file('ticker-cfg.yaml') + + assert chk1 == chk2, f'\nOUTPUT 1\n{chk1}\n!= OUTPUT 2\n{chk2}\n' + return t + + def ticker31(self): + t = self.ticker(add_opts=['--wide']) + chk1 = '\n'.join(t.read().splitlines()[5:-2]) + + t = self.ticker(add_opts=['--wide', '--pchg-unit=usd'], no_msg=True) + chk2 = '\n'.join(t.read().splitlines()[6:-2]) + + assert chk1 == chk2, f'\nOUTPUT 1\n{chk1}\n!= OUTPUT 2\n{chk2}\n' + return t + + def ticker32(self): + return self.ticker( + [], + [ + 'BITCOIN', r'\+10.99', r'\+7.06', '-1.18', r'\+1.05', + 'ETHEREUM', + 'GOLD', r'\+0.00', r'\+0.00', r'\+0.00', r'\+0.00', + 'SILVER' + ], + add_opts = ['--widest', '--pchg-unit=gc=f']) diff --git a/test/ref/ticker/ticker-cfg-sort-pchg.yaml b/test/ref/ticker/ticker-cfg-sort-pchg.yaml new file mode 100644 index 0000000..4660ae4 --- /dev/null +++ b/test/ref/ticker/ticker-cfg-sort-pchg.yaml @@ -0,0 +1,14 @@ +sort: d +pchg_unit: btc + +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + - bad-badcoin + commodity: + - gc=f + - si=f + - bz=f From d02c9936ef7504e70b07e82401042ae5fc58901a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 22 Oct 2025 10:33:16 +0000 Subject: [PATCH 104/108] mmnode-ticker: fix market cap sort with USD row --- mmgen_node_tools/Ticker.py | 1 + mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 26 +++++++++++++++++++++++++- test/ref/ticker/ticker-cfg-usd.yaml | 11 +++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-usd.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 4b49b44..15e0e9c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -539,6 +539,7 @@ def gen_data(data): 'percent_change_7d': 0.0, 'percent_change_30d': 0.0, 'percent_change_1y': 0.0, + 'market_cap': 0, 'last_updated': None}) def cache_data(data_src, no_overwrite=False): diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 47203f7..bd10d35 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev9 +3.6.dev10 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9112f48..d5cb51d 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -101,6 +101,9 @@ class CmdTestScripts(CmdTestBase): ('ticker30', 'ticker [--cached-data --wide --pchg-unit=btc --sort=d] (cf with config file)'), ('ticker31', 'ticker [--cached-data --wide --pchg-unit=usd] (cf with no USD)'), ('ticker32', 'ticker [--cached-data --wide --pchg-unit=gc=f]'), + ('ticker33', 'ticker [--cached-data --wide --pchg-unit=btc --sort=c] (cfg file with USD)'), + ('ticker34', 'ticker [--cached-data --wide --pchg-unit=btc --sort=y] (cfg file with USD)'), + ('ticker35', 'ticker [--cached-data --wide --pchg-unit=btc --sort=p] (cfg file with USD)'), ) } @@ -475,4 +478,25 @@ class CmdTestScripts(CmdTestBase): 'GOLD', r'\+0.00', r'\+0.00', r'\+0.00', r'\+0.00', 'SILVER' ], - add_opts = ['--widest', '--pchg-unit=gc=f']) + add_opts = ['--widest', '--pchg-unit=gc=f', '--sort=c']) + + def _ticker_cur(self, sort): + self.copy_file('ticker-cfg-usd.yaml', 'ticker-cfg.yaml') + t = self.ticker( + [], + [ + 'BITCOIN 23,250.77 1.00000000 \+0.00', + 'US DOLLAR 1.00 0.00004301 -15.93', + ], + add_opts = ['--widest', '--pchg-unit=btc', f'--sort={sort}']) + self.rm_file('ticker-cfg.yaml') + return t + + def ticker33(self): + return self._ticker_cur(sort='c') + + def ticker34(self): + return self._ticker_cur(sort='y') + + def ticker35(self): + return self._ticker_cur(sort='p') diff --git a/test/ref/ticker/ticker-cfg-usd.yaml b/test/ref/ticker/ticker-cfg-usd.yaml new file mode 100644 index 0000000..c31f85f --- /dev/null +++ b/test/ref/ticker/ticker-cfg-usd.yaml @@ -0,0 +1,11 @@ +assets: + coin1: + - btc-bitcoin + - eth-ethereum + - xmr-monero + commodity: + - gc=f + - si=f + currency: + - usd-us-dollar + - eurusd=x From f5c165c36201c0b251298fd08cef9566c42b37ee Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:31 +0000 Subject: [PATCH 105/108] whitespace, minor cleanups and fixes --- mmgen_node_tools/BlocksInfo.py | 5 +- mmgen_node_tools/Ticker.py | 3 +- test/cmdtest_d/regtest.py | 110 +++++++++++++++++---------------- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index a59bf50..c068017 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -353,7 +353,8 @@ class BlocksInfo: nblocks, step = (add1, add2) if last is None else (None, add1) - if p.debug: msg(repr(self.range_data(first, last, from_tip, nblocks, step))) + if p.debug: + msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if nblocks: if first is None: @@ -643,9 +644,7 @@ class BlocksInfo: values = {n: d.func(n) for n in fnames} col1_w = max((len(l) for l in lbls.values()), default=0) + 2 - print(d.spec_vals) for v in d.spec_vals: - print(v) if v.condition(values): try: idx = fnames.index(v.insert_after) + 1 except: idx = 0 diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 15e0e9c..50bb85b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -970,8 +970,7 @@ class Ticker: t = int(d[row.id]['last_updated']) except TypeError as e: d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) - except KeyError as e: - msg(str(e)) + except KeyError: pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index a504678..b35ec7a 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -23,18 +23,18 @@ from ..include.common import cfg, imsg, stop_test_daemons, joinpath from .base import CmdTestBase args1 = ['--bob'] -args2 = ['--bob','--rpc-backend=http'] +args2 = ['--bob', '--rpc-backend=http'] -def gen_addrs(proto,network,keys): +def gen_addrs(proto, network, keys): from mmgen.tool.api import tool_api tool = tool_api(cfg) - tool.init_coin(proto.coin,'regtest') + tool.init_coin(proto.coin, 'regtest') tool.addrtype = proto.mmtypes[-1] return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' - networks = ('btc','ltc','bch') + networks = ('btc', 'ltc', 'bch') passthru_opts = ('coin',) tmpdir_nums = [1] color = True @@ -110,31 +110,31 @@ class CmdTestRegtest(CmdTestBase): if trunner is None: return if cfg._proto.testnet: - die(2,'--testnet and --regtest options incompatible with regtest test suite') + die(2, '--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) - self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto,'regtest',[1,2,3,4,5])] + self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto, 'regtest', [1, 2, 3, 4, 5])] self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): - stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) + stop_test_daemons(self.proto.network_id, force=True, remove_datadir=True) from shutil import rmtree try: - rmtree(joinpath(self.tr.data_dir,'regtest')) + rmtree(joinpath(self.tr.data_dir, 'regtest')) except: pass t = self.spawn( 'mmgen-regtest', (['--bdb-wallet'] if self.use_bdb_wallet else []) + ['--setup-no-stop-daemon', 'setup']) - for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): + for s in ('Starting', 'Creating', 'Creating', 'Creating', 'Mined', 'Setup complete'): t.expect(s) return t def netrate(self, add_args, expect_str, exit_val=None): t = self.spawn('mmnode-netrate', args1 + add_args, exit_val=exit_val) - t.expect(expect_str,regex=True) + t.expect(expect_str, regex=True) return t def netrate1(self): @@ -147,35 +147,35 @@ class CmdTestRegtest(CmdTestBase): return 'ok' return t - def halving_calculator(self,add_args,expect_list): - t = self.spawn('mmnode-halving-calculator',args1+add_args) + def halving_calculator(self, add_args, expect_list): + t = self.spawn('mmnode-halving-calculator', args1+add_args) t.match_expect_list(expect_list) return t def halving_calculator1(self): - return self.halving_calculator(['--help'],['USAGE:']) + return self.halving_calculator(['--help'], ['USAGE:']) def halving_calculator2(self): - return self.halving_calculator([],['Current block: 393',f'Current block subsidy: 12.5 {cfg.coin}']) + return self.halving_calculator([], ['Current block: 393', f'Current block subsidy: 12.5 {cfg.coin}']) def halving_calculator3(self): - return self.halving_calculator(['--list'],['33 4950','0']) + return self.halving_calculator(['--list'], ['33 4950', '0']) def halving_calculator4(self): - return self.halving_calculator(['--mined'],['0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined'], ['0 0.0000015 14949.9999835']) def halving_calculator5(self): - return self.halving_calculator(['--mined','--bdr-proj=5'],['5.00000 0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined', '--bdr-proj=5'], ['5.00000 0 0.0000015 14949.9999835']) def halving_calculator6(self): - return self.halving_calculator(['--mined','--sample-size=20'],['33 4950','0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined', '--sample-size=20'], ['33 4950', '0 0.0000015 14949.9999835']) - def sendto(self,addr,amt): - return self.spawn('mmgen-regtest',['send',addr,amt]) + def sendto(self, addr, amt): + return self.spawn('mmgen-regtest', ['send', addr, amt]) - def sendto1(self): return self.sendto(self.addrs[0],'0.123') - def sendto2(self): return self.sendto(self.addrs[0],'0.234') - def sendto3(self): return self.sendto(self.addrs[1],'0.345') + def sendto1(self): return self.sendto(self.addrs[0], '0.123') + def sendto2(self): return self.sendto(self.addrs[0], '0.234') + def sendto3(self): return self.sendto(self.addrs[1], '0.345') def addrbal(self, args, expect_list): t = self.spawn('mmnode-addrbal', args2 + args) @@ -231,7 +231,7 @@ class CmdTestRegtest(CmdTestBase): [ 'No balance', '2 unspent outputs in 2 blocks', - '394','0.123','395','0.234', + '394', '0.123', '395', '0.234', 'No balance' ]) @@ -240,7 +240,7 @@ class CmdTestRegtest(CmdTestBase): ['--tabular', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - -', - self.addrs[0] + ' 2 395','0.357', + self.addrs[0] + ' 2 395', '0.357', self.addrs[3] + ' - - -', ]) @@ -249,11 +249,11 @@ class CmdTestRegtest(CmdTestBase): ['--tabular', '--first-block', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - - -', - self.addrs[0] + ' 2 394','395','0.357', + self.addrs[0] + ' 2 394', '395', '0.357', self.addrs[3] + ' - - - -', ]) - def blocks_info(self,args,expect_list): + def blocks_info(self, args, expect_list): t = self.spawn('mmnode-blocks-info', args1 + args) t.match_expect_list(expect_list) return t @@ -261,7 +261,7 @@ class CmdTestRegtest(CmdTestBase): def blocks_info1(self): return self.blocks_info( ['--help'], - ['USAGE:','OPTIONS:']) + ['USAGE:', 'OPTIONS:']) def blocks_info2(self): return self.blocks_info( @@ -278,7 +278,7 @@ class CmdTestRegtest(CmdTestBase): ]) def blocks_info4(self): - n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) + n1, i1, o1, n2, i2, o2 = (2, 1, 3, 6, 3, 9) if cfg.coin == 'BCH' else (2, 1, 4, 6, 3, 12) return self.blocks_info( ['--miner-info', '--fields=all', '--stats=all', '+3'], [ @@ -302,21 +302,21 @@ class CmdTestRegtest(CmdTestBase): from collections import namedtuple t = tool_api(cfg) - t.init_coin(self.proto.coin,self.proto.network) + t.init_coin(self.proto.coin, self.proto.network) t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32' - wp = namedtuple('wifaddrpair',['wif','addr']) + wp = namedtuple('wifaddrpair', ['wif', 'addr']) def gen(): - for n in range(0xfaceface,nPairs+0xfaceface): + for n in range(0xfaceface, nPairs+0xfaceface): wif = t.hex2wif(f'{n:064x}') yield wp( wif, t.wif2addr(wif) ) return list(gen()) - def gen_fees(n_in,low,high): + def gen_fees(n_in, low, high): # very approximate tx size estimation: - ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) + ibytes, wbytes, obytes = (148, 0, 34) if self.proto.coin == 'BCH' else (43, 108, 31) x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt.satoshi n = n_in - 1 @@ -325,14 +325,20 @@ class CmdTestRegtest(CmdTestBase): for i in range(n_in): yield Decimal(low + (i/n)**6 * vmax) * x - async def do_tx(inputs,outputs,wif): - tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) - tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) + async def do_tx(inputs, outputs, wif): + tx_hex = await r.rpc_call('createrawtransaction', inputs, outputs) + if wif: + tx = await r.rpc_call( + 'signrawtransactionwithkey', + tx_hex, + [wif], + [], + self.proto.sighash_type) assert tx['complete'] return tx['hex'] async def do_tx1(): - us = await r.rpc_call('listunspent',wallet='miner') + us = await r.rpc_call('listunspent', wallet='miner') tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr: tx1_amt for p in pairs[:nTxs]} @@ -342,7 +348,7 @@ class CmdTestRegtest(CmdTestBase): outputs, await r.miner_wif) - async def do_tx2(tx,pairno): + async def do_tx2(tx, pairno): fee = self.proto.coin_amt(fees[pairno], from_decimal=True) outputs = {p.addr: tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) @@ -353,10 +359,10 @@ class CmdTestRegtest(CmdTestBase): async def do_txs(tx_in): for pairno in range(nTxs): - tx_hex = await do_tx2(tx_in,pairno) - await r.rpc_call('sendrawtransaction',tx_hex) + tx_hex = await do_tx2(tx_in, pairno) + await r.rpc_call('sendrawtransaction', tx_hex) - self.spawn('',msg_only=True) + self.spawn('', msg_only=True) r = self.regtest nPairs = 100 @@ -372,20 +378,20 @@ class CmdTestRegtest(CmdTestBase): tx1_hex = await do_tx1() imsg('Relaying funding transaction') - await r.rpc_call('sendrawtransaction',tx1_hex) + await r.rpc_call('sendrawtransaction', tx1_hex) imsg('Mining a block') - await r.generate(1,silent=True) + await r.generate(1, silent=True) imsg('Generating fees for mempool transactions') - fees = list(gen_fees(nTxs,2,120)) + fees = list(gen_fees(nTxs, 2, 120)) imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') - await do_txs(await r.rpc_call('decoderawtransaction',tx1_hex)) + await do_txs(await r.rpc_call('decoderawtransaction', tx1_hex)) return 'ok' - def _feeview(self,args,expect_list=[]): + def _feeview(self, args, expect_list=[]): t = self.spawn('mmnode-feeview', args1 + args) if expect_list: t.match_expect_list(expect_list) @@ -395,7 +401,7 @@ class CmdTestRegtest(CmdTestBase): return self._feeview([]) def feeview2(self): - return self._feeview(['--columns=40','--include-current']) + return self._feeview(['--columns=40', '--include-current']) def feeview3(self): return self._feeview(['--precision=6']) @@ -404,7 +410,7 @@ class CmdTestRegtest(CmdTestBase): return self._feeview(['--detail']) def feeview5(self): - return self._feeview(['--show-empty','--log',f'--outdir={self.tmpdir}']) + return self._feeview(['--show-empty', '--log', f'--outdir={self.tmpdir}']) def feeview6(self): return self._feeview(['--ignore-below=1MB']) @@ -414,13 +420,13 @@ class CmdTestRegtest(CmdTestBase): async def feeview8(self): imsg('Clearing mempool') - await self.regtest.generate(1,silent=True) + await self.regtest.generate(1, silent=True) return self._feeview([]) def stop(self): if cfg.no_daemon_stop: - self.spawn('',msg_only=True) + self.spawn('', msg_only=True) msg_r('(leaving daemon running by user request)') return 'ok' else: - return self.spawn('mmgen-regtest',['stop']) + return self.spawn('mmgen-regtest', ['stop']) From e5bc33a6d6267ec320ddf4ae3511fc4590e4ec1b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:32 +0000 Subject: [PATCH 106/108] update for MMGen Wallet v16.1.dev21 --- mmgen_node_tools/main_addrbal.py | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 4647029..743b36e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -44,7 +44,7 @@ def do_output(proto, addr_data, blk_hdrs): heights = {u['height'] for u in unspents} Msg('{}Balance: {}'.format( indent, - sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), + sum(proto.coin_amt(u['amount']) for u in unspents).hl3(unit=True, fs='{:,}'))), Msg('{}{} unspent output{} in {} block{}'.format( indent, red(str(len(unspents))), @@ -148,7 +148,7 @@ async def main(req_addrs): good_addrs = len([v for v in addr_data.values() if v]) Msg('Total: {} in {} address{}'.format( - proto.coin_amt(res['total_amount']).hl2(unit=True, fs='{:,}'), + proto.coin_amt(res['total_amount']).hl3(unit=True, fs='{:,}'), red(str(good_addrs)), suf(good_addrs, 'es'))) diff --git a/setup.cfg b/setup.cfg index d48a8c8..91b63b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev4 + mmgen-wallet>=16.1.dev21 pyyaml yahooquery From b97b4f5f634f2498947f4ea4b25f5ac71d08db85 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:32 +0000 Subject: [PATCH 107/108] update for Bitcoin Core v30.0, nixpkgs 25.11 --- mmgen_node_tools/data/version | 2 +- nix/curl-cffi.nix | 50 ----------------------------------- nix/yahooquery.nix | 2 +- test/cmdtest_d/regtest.py | 9 ++++++- 4 files changed, 10 insertions(+), 53 deletions(-) delete mode 100644 nix/curl-cffi.nix diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index bd10d35..4e5cab0 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev10 +3.6.dev11 diff --git a/nix/curl-cffi.nix b/nix/curl-cffi.nix deleted file mode 100644 index 15f9764..0000000 --- a/nix/curl-cffi.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ - stdenv, - lib, - python, - buildPythonPackage, - fetchPypi, - curl-impersonate-chrome, -}: - -buildPythonPackage rec { - pname = "curl-cffi"; - # version = "0.13.0"; # uses option PROXY_CREDENTIAL_NO_REUSE, unavailable in current libcurl - version = "0.10.0"; - pyproject = true; - - src = fetchPypi { - pname = "curl_cffi"; - version = version; - # hash = "sha256-YuzZCjgr1QI3UONgbgqnyxo6i6QcFCcLjl4Unr9yxco="; # 0.13.0 - hash = "sha256-PjezUmjKWEkvVO0CCuS1DDPuDeutQUXbn3RvBO1GbrA="; # 0.10.0 - }; - - patches = [ ./use-system-libs.patch ]; - - buildInputs = [ curl-impersonate-chrome ]; - - build-system = with python.pkgs; [ - cffi - setuptools - ]; - - dependencies = with python.pkgs; [ - cffi - certifi - typing-extensions - ]; - - env = lib.optionalAttrs stdenv.cc.isGNU { - NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types"; - }; - - pythonImportsCheck = [ "curl_cffi" ]; - - meta = with lib; { - description = "Python binding for curl-impersonate via cffi"; - homepage = "https://curl-cffi.readthedocs.io"; - license = licenses.mit; - maintainers = with maintainers; [ chuangzhu ]; - }; -} diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix index 3e8925f..ecb2494 100644 --- a/nix/yahooquery.nix +++ b/nix/yahooquery.nix @@ -19,7 +19,7 @@ buildPythonPackage rec { build-system = with python.pkgs; [ hatchling ]; propagatedBuildInputs = with python.pkgs; [ - (callPackage ./curl-cffi.nix {}) # >=0.10.0 + curl-cffi pandas requests-futures tqdm diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index b35ec7a..387594b 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -114,7 +114,7 @@ class CmdTestRegtest(CmdTestBase): self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto, 'regtest', [1, 2, 3, 4, 5])] - self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' + self.use_bdb_wallet = self.bdb_wallet and self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): @@ -334,6 +334,13 @@ class CmdTestRegtest(CmdTestBase): [wif], [], self.proto.sighash_type) + else: + tx = await r.rpc_call( + 'signrawtransactionwithwallet', + tx_hex, + None, # prevtxs + self.proto.sighash_type, + wallet = 'miner') assert tx['complete'] return tx['hex'] From f88b70c2e2413d6261dcc3d5d5d930ec778d9eff Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 27 Jan 2026 08:46:15 +0000 Subject: [PATCH 108/108] mmnode-ticker: support nested assets in `ticker-portfolio.yaml` --- mmgen_node_tools/Ticker.py | 39 ++++++++++++++------- mmgen_node_tools/data/ticker-portfolio.yaml | 10 +++++- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/ref/ticker/ticker-portfolio.yaml | 9 +++-- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 50bb85b..a0adb39 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -720,15 +720,28 @@ def make_cfg(gcfg_arg): + usr_columns) def get_portfolio_assets(): - if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e) for e in cfg_in.portfolio) + if portfolio: + ret = (parse_asset_id(e) for e in portfolio) return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') else: return () - def get_portfolio(): - return tuple((k, Decimal(v)) for k, v in cfg_in.portfolio.items() - if (not gcfg.btc) or k == 'btc-bitcoin') + def parse_portfolio(): + ret = {} + def add(k, v): + if gcfg.btc and k != 'btc-bitcoin': + return + if k in ret: + ret[k] += Decimal(v) + else: + ret[k] = Decimal(v) + for k, v in cfg_in.portfolio.items(): + if isinstance(v, dict): + for k2, v2 in v.items(): + add(k2, v2) + else: + add(k, v) + return ret def parse_add_precision(arg): if not arg: @@ -834,7 +847,7 @@ def make_cfg(gcfg_arg): proxy2 = get_proxy('proxy2') portfolio = ( - get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query + parse_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query else None) if portfolio and asset_range: @@ -928,13 +941,14 @@ class Ticker: key = sort_params[code].key sort_dfl = sort_params[code].sort_dfl sort_func = lambda row: data.get(row.id, {key: sort_dfl})[key] - pf_sort_func = lambda row: data.get(row[0], {key: sort_dfl})[key] + pf_sort_func = lambda row: data.get(row, {key: sort_dfl})[key] for group in self.rows.keys(): if group not in self.hidden_groups: self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) if cfg.portfolio: - cfg = cfg._replace( - portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + cfg = cfg._replace(portfolio = + {k: cfg.portfolio[k] + for k in sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)}) if cfg.pchg_unit: self.pchg_data = self.data[self.get_id(cfg.pchg_unit)] @@ -1061,7 +1075,7 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym, amt in cfg.portfolio: + for sym, amt in cfg.portfolio.items(): try: yield self.fmt_row(self.data[sym], amt=amt) except KeyError: @@ -1086,10 +1100,9 @@ class Ticker: self.format_last_updated_col() if cfg.portfolio: - pf_dict = dict(cfg.portfolio) - self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * pf_dict[row.id] + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] for row in self.rows - if row.id in pf_dict and row.id in data) + if row.id in cfg.portfolio and row.id in data) for col_id in self.col_ids} self.init_prec() diff --git a/mmgen_node_tools/data/ticker-portfolio.yaml b/mmgen_node_tools/data/ticker-portfolio.yaml index fcadd30..b63e626 100644 --- a/mmgen_node_tools/data/ticker-portfolio.yaml +++ b/mmgen_node_tools/data/ticker-portfolio.yaml @@ -3,4 +3,12 @@ # Invoke `mmnode-ticker --list-ids` for a full list of supported asset IDs. btc-bitcoin: '1.23456789' eth-ethereum: '2.3456789012' -xmr-monero: '4.5678901234' +xmr-monero: '4.567890123456' + +# Nested values are supported. Values for each asset will be summed. +wallet2: + btc-bitcoin: '0.12345678' + ltc-litecoin: '1.23456789' + +exchange1: + xmr-monero: '12.345678901234' diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4e5cab0..6b99850 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev11 +3.6.dev12 diff --git a/setup.cfg b/setup.cfg index 91b63b2..0753d4f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev21 + mmgen-wallet>=16.1.dev26 pyyaml yahooquery diff --git a/test/ref/ticker/ticker-portfolio.yaml b/test/ref/ticker/ticker-portfolio.yaml index 332257f..ca0c2b2 100644 --- a/test/ref/ticker/ticker-portfolio.yaml +++ b/test/ref/ticker/ticker-portfolio.yaml @@ -1,5 +1,10 @@ btc-bitcoin: '1.23456789' eth-ethereum: '2.345678901234567890' -xmr-monero: '4.567890123456' +xmr-monero: '3.333390123456' ada-cardano: '123.45678901' -algo-algorand: '234.5678901' + +wallet2: + algo-algorand: '234.5678901' + +exchange1: + xmr-monero: '1.2345'