From 6074a0e42a4b2b88e07ed47584553eb4c7b76125 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 25 Sep 2023 15:53:02 +0000 Subject: [PATCH 01/81] mmnode-ticker: fixes and cleanups throughout --- mmgen_node_tools/Ticker.py | 120 ++++++++++++++++---------- mmgen_node_tools/data/ticker-cfg.yaml | 13 +-- mmgen_node_tools/main_ticker.py | 28 +++--- test/test_py_d/ts_misc.py | 8 +- 4 files changed, 102 insertions(+), 67 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index e4b147c..a370b56 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -28,11 +28,12 @@ import sys,os,time,json,yaml from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple + from mmgen.color import * -from mmgen.util import die,fmt_list,msg,msg_r,suf,fmt +from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list homedir = os.getenv('HOME') -cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') +dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' @@ -78,7 +79,7 @@ def gen_data(data): die(1,'Missing data, exiting') rows_want = { - 'id': {r.id for r in cfg.rows if getattr(r,'id',None)} - {'usd-us-dollar'}, + 'id': {r.id for r in cfg.rows if isinstance(r,tuple) and r.id} - {'usd-us-dollar'}, 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, } usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) @@ -100,25 +101,29 @@ def gen_data(data): found = { 'id': set(), 'symbol': set() } rate_assets = {} - for k in ['id','symbol']: - wants = rows_want[k] | usr_wants[k] - if wants: - for d in data: - if d[k] in wants: - if d[k] in found[k]: - die(1,dup_sym_errmsg(d[k])) - yield (d['id'],d) - found[k].add(d[k]) - if d[k] in usr_rate_assets_want[k]: - rate_assets[d['symbol']] = d # NB: using symbol instead of ID - if k == 'id' and len(found[k]) == len(wants): - break + wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} for d in data: if d['id'] == 'btc-bitcoin': btcusd = Decimal(d['price_usd']) break + for k in ('id','symbol'): + for d in data: + if wants[k]: + if d[k] in wants[k]: + if d[k] in found[k]: + die(1,dup_sym_errmsg(d[k])) + yield (d['id'],d) + found[k].add(d[k]) + wants[k].remove(d[k]) + if d[k] in usr_rate_assets_want[k]: + rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key + else: + break + + check_assets_found(usr_wants,found) + for asset in (cfg.usr_rows + cfg.usr_columns): if asset.rate: """ @@ -129,21 +134,21 @@ def gen_data(data): yield ( _id, { 'symbol': asset.symbol, 'id': _id, + 'name': ' '.join(_id.split('-')[1:]), 'price_usd': str(Decimal(ra_rate/asset.rate)), 'price_btc': str(Decimal(ra_rate/asset.rate/btcusd)), - 'last_updated': int(now), + 'last_updated': None, }) yield ('usd-us-dollar', { 'symbol': 'USD', 'id': 'usd-us-dollar', + 'name': 'US Dollar', 'price_usd': '1.0', 'price_btc': str(Decimal(1/btcusd)), - 'last_updated': int(now), + 'last_updated': None, }) - check_assets_found(usr_wants,found) - def get_src_data(curl_cmd): tor_captcha_msg = f""" @@ -166,8 +171,8 @@ def get_src_data(curl_cmd): ('' if cfg.btc_only else ', or use --cached-data or --btc') ) - if not os.path.exists(cachedir): - os.makedirs(cachedir) + if not os.path.exists(cfg.cachedir): + os.makedirs(cfg.cachedir) if cfg.btc_only: fn = os.path.join(cfg.cachedir,'ticker-btc.json') @@ -225,18 +230,18 @@ def get_src_data(curl_cmd): return data -def main(cfg_parm,cfg_in_parm): +def main(): def update_sample_file(usr_cfg_file): - src_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() + usr_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() sample_file = usr_cfg_file + '.sample' sample_data = open(sample_file).read() if os.path.exists(sample_file) else None - if src_data != sample_data: + if usr_data != sample_data: os.makedirs(os.path.dirname(sample_file),exist_ok=True) msg('{} {}'.format( ('Updating','Creating')[sample_data is None], sample_file )) - open(sample_file,'w').write(src_data) + open(sample_file,'w').write(usr_data) def get_curl_cmd(): return ([ @@ -251,8 +256,6 @@ def main(cfg_parm,cfg_in_parm): ) global cfg,cfg_in - cfg = cfg_parm - cfg_in = cfg_in_parm try: from importlib.resources import files # Python 3.9 @@ -272,21 +275,23 @@ def main(cfg_parm,cfg_in_parm): Msg(curl_cmd + '\n' + ' '.join(curl_cmd)) return - parsed_json = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) + src_data = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) if gcfg.list_ids: from mmgen.ui import do_pager - do_pager('\n'.join(e['id'] for e in parsed_json)) + do_pager('\n'.join(e['id'] for e in src_data)) return global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 + data = dict(gen_data(src_data)) + gcfg._util.stdout_or_pager( - '\n'.join(getattr(Ticker,cfg.clsname)(dict(gen_data(parsed_json))).gen_output()) + '\n' + '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n' ) -def make_cfg(cmd_args,cfg_in): +def make_cfg(): def get_rows_from_cfg(add_data=None): def gen(): @@ -295,7 +300,7 @@ def make_cfg(cmd_args,cfg_in): if add_data and k in add_data: v += tuple(add_data[k]) for e in v: - yield parse_asset_id(e,True) + yield parse_asset_id(e,require_label=True) return tuple(gen()) def parse_asset_id(s,require_label=False): @@ -304,7 +309,7 @@ def make_cfg(cmd_args,cfg_in): die(1,f'{s!r}: asset label is missing') return asset_tuple( sym.upper(), (s.lower() if label else None) ) - def parse_usr_asset_arg(s): + def parse_usr_asset_arg(key,use_cf_file=False): """ asset_id[:rate[:rate_asset]] """ @@ -324,7 +329,9 @@ def make_cfg(cmd_args,cfg_in): Decimal(rate) ), rate_asset = parse_asset_id(rate_asset) if rate_asset else None ) - return tuple(parse_parm(s2) for s2 in s.split(',')) if s else () + cl_opt = getattr(gcfg,key) + cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] + return tuple( parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt) ) def parse_query_arg(s): """ @@ -365,7 +372,7 @@ def make_cfg(cmd_args,cfg_in): def get_portfolio_assets(ret=()): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e,True) for e in cfg_in.portfolio) + ret = (parse_asset_id(e,require_label=True) for e in cfg_in.portfolio) return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') ) def get_portfolio(): @@ -410,15 +417,30 @@ def make_cfg(cmd_args,cfg_in): 'proxy', 'portfolio' ]) + global cfg_in,cfg + + cmd_args = gcfg._args + cfg_in = get_cfg_in() + query_tuple = namedtuple('query',['asset','to_asset']) asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset']) asset_tuple = namedtuple('asset_tuple',['symbol','id']) - usr_rows = parse_usr_asset_arg(gcfg.add_rows) - usr_columns = parse_usr_asset_arg(gcfg.add_columns) + usr_rows = parse_usr_asset_arg('add_rows') + usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None - return cfg_tuple( + def get_proxy(name): + proxy = getattr(gcfg,name) + return ( + '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') + else (proxy or cfg_in.cfg.get(name)) + ) + + proxy = get_proxy('proxy') + proxy = None if proxy == 'none' else proxy + + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, usr_columns = usr_columns, @@ -427,8 +449,8 @@ def make_cfg(cmd_args,cfg_in): clsname = 'trading' if query else 'overview', btc_only = gcfg.btc, add_prec = parse_add_precision(gcfg.add_precision), - cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or cachedir, - proxy = None if gcfg.proxy == '' else (gcfg.proxy or cfg_in.cfg.get('proxy')), + cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, + proxy = proxy, portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None ) @@ -489,13 +511,21 @@ class Ticker: d = self.data max_w = 0 - min_t = min( (int(d[a.id]['last_updated']) for a in cross_assets), default=None ) + + if cross_assets: + last_updated_x = [d[a.id]['last_updated'] for a in cross_assets] + min_t = min( (int(n) for n in last_updated_x if isinstance(n,int) ), default=None ) + else: + min_t = None for row in self.rows: if isinstance(row,tuple): try: - t = int(d[row.id]['last_updated']) - except KeyError: + t = int( d[row.id]['last_updated'] ) + except TypeError as e: + d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) + except KeyError as e: + msg(str(e)) pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now ) @@ -517,7 +547,7 @@ class Ticker: return d['id'] def create_label(self,id): - return ' '.join(id.split('-')[1:]).upper() + return self.data[id]['name'].upper() def gen_output(self): yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now))) diff --git a/mmgen_node_tools/data/ticker-cfg.yaml b/mmgen_node_tools/data/ticker-cfg.yaml index 2566da4..6589bfc 100644 --- a/mmgen_node_tools/data/ticker-cfg.yaml +++ b/mmgen_node_tools/data/ticker-cfg.yaml @@ -1,14 +1,15 @@ -# Indentation must be consistent! Do not mix leading tabs and spaces. +### Indentation must be consistent! Do not mix leading tabs and spaces. -# See the curl manpage for supported --proxy parameters -# For a direct connection, leave the right-hand side blank +### See the curl manpage for supported --proxy parameters +### For a direct connection, leave the right-hand side blank proxy: http://vpn-gw:8118 -# Override the default cache directory (~/.cache/mmgen-node-tools) +### Override the default cache directory (~/.cache/mmgen-node-tools): cachedir: -# Asset labels are arbitrary strings. Use as many or few as you wish. -# Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. +### Asset rows +### Asset labels are arbitrary strings. Use as many or few as you wish. +### Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. assets: coin1: - btc-bitcoin diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 83d0b1c..b8df683 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -12,7 +12,6 @@ mmnode-ticker: Display price information for cryptocurrency and other assets """ -import sys,os from .Ticker import * opts_data = { @@ -36,8 +35,8 @@ opts_data = { used to supply a USD exchange rate for missing assets. -C, --cached-data Use cached data from previous network query instead of live data from server --d, --cachedir=D Read and write cached JSON data to directory ‘D’ - instead of ‘~/{os.path.relpath(cachedir,start=homedir)}’ +-D, --cachedir=D Read and write cached JSON data to directory ‘D’ + instead of ‘~/{dfl_cachedir}’ -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -48,13 +47,14 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. +-t, --testing Print command to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column --U, --print-curl Print cURL command to standard output and exit -v, --verbose Be more verbose -w, --wide Display all optional columns (equivalent to -punT) -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to - disable. Consult the curl manpage for --proxy usage. + completely disable or ‘none’ to allow override from + environment. Consult the curl manpage for --proxy usage. """, 'notes': """ @@ -184,6 +184,9 @@ To add a portfolio, edit the file """ }, 'code': { + 'options': lambda s: s.format( + dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), + ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), @@ -196,16 +199,17 @@ To add a portfolio, edit the file } } +import os + from mmgen.cfg import Config -gcfg = Config( opts_data=opts_data, do_post_init=True ) +import mmgen_node_tools.Ticker as tck -import mmgen_node_tools.Ticker as Ticker -Ticker.gcfg = gcfg +tck.gcfg = Config( opts_data=opts_data, do_post_init=True ) -cfg_in = get_cfg_in() +tck.make_cfg() -cfg = make_cfg(gcfg._args,cfg_in) +from .Ticker import cfg_in -gcfg._post_init() +tck.gcfg._post_init() -main(cfg,cfg_in) +tck.main() diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index c4536d6..c1e925d 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -198,7 +198,7 @@ class TestSuiteScripts(TestSuiteBase): [ 'USD BTC CHG_7d CHG_24h UPDATED', r'BITCOIN 23,250.7741 1.0000000000 \+11.15 \+0.89 10 minutes ago', - r'FAKECOIN 81.3008 0.0034966927 -- -- just now', + r'FAKECOIN 81.3008 0.0034966927 -- -- --', r'\(no data for noc-nocoin\)', ]) os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) @@ -247,7 +247,7 @@ class TestSuiteScripts(TestSuiteBase): 'Offer: 200,000 INR', 'Offered price differs from spot by -7.58%', 'SPOT PRICE OFFERED PRICE UPDATED', - 'INDIAN RUPEE 200,000.00000000 184,843.65372424 10 minutes ago ' + + 'INDIAN RUPEE 200,000.00000000 184,843.65372424 -- ' + 'BITCOIN 0.10819955 0.10000000 10 minutes ago' ]) @@ -272,7 +272,7 @@ class TestSuiteScripts(TestSuiteBase): 'Offered price differs from spot by -3.72%', 'SPOT PRICE OFFERED PRICE UPDATED', 'BITCOIN 2.00000000 1.92563954 1 day 9 hours 2 minutes ago ' + - 'US DOLLAR 46,737.71911598 45,000.00000000 1 day 9 hours 2 minutes ago', + 'US DOLLAR 46,737.71911598 45,000.00000000 --', ]) def ticker16(self): @@ -283,7 +283,7 @@ class TestSuiteScripts(TestSuiteBase): r'OMR \(OMANI RIAL\) = 2.5900 USD', 'USD EUR OMR BTC CHG_7d CHG_24h UPDATED', r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', - 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- just now' + 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- --' ]) def ticker17(self): From d29e34c221b8f95099c6f56edb62bf7fc2b0ce95 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 25 Sep 2023 15:53:02 +0000 Subject: [PATCH 02/81] mmnode-ticker: support multiple data sources --- mmgen_node_tools/Ticker.py | 317 ++++++++++++++++++++------------ mmgen_node_tools/main_ticker.py | 29 ++- 2 files changed, 210 insertions(+), 136 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index a370b56..cafd407 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -12,11 +12,6 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets """ -api_host = 'api.coinpaprika.com' -api_url = f'https://{api_host}/v1/ticker' -ratelimit = 240 -btc_ratelimit = 10 - # We use deprecated coinpaprika ‘ticker’ API for now because it returns ~45% less data. # Old ‘ticker’ API (/v1/ticker): data['BTC']['price_usd'] # New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price'] @@ -24,25 +19,193 @@ btc_ratelimit = 10 # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,time,json,yaml +import sys,os,re,time,json,yaml,random from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple from mmgen.color import * -from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list +from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen homedir = os.getenv('HOME') dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' +asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) + +def fetch_delay(fetched_data=[]): + if not gcfg.testing: + if fetched_data: + delay = 1 + random.randrange(1,5000) / 1000 + msg(f'Waiting {delay:.3f} seconds...') + time.sleep(delay) + else: + fetched_data.append(None) + +class DataSource: + + sources = { + 'cc': 'coinpaprika', + } + + class base: + + def get_data_from_network(self): + + curl_cmd = list_gen( + ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], + ['--compressed'], # adds 'Accept-Encoding: gzip' + ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], + ['--silent', not gcfg.verbose], + [self.api_url] + ) + + if gcfg.testing: + Msg(fmt_list(curl_cmd,fmt='bare')) + return + + try: + return run(curl_cmd,check=True,stdout=PIPE).stdout.decode() + except CalledProcessError as e: + msg('') + from .Misc import curl_exit_codes + msg(red(curl_exit_codes[e.returncode])) + msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) + from mmgen.exception import MMGenCalledProcessError + raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') + + def get_data(self): + + if not os.path.exists(cfg.cachedir): + os.makedirs(cfg.cachedir) + + if not os.path.exists(self.json_fn): + open(self.json_fn,'w').write('{}') + + if gcfg.cached_data: + data_type = 'json' + data_in = open(self.json_fn).read() + else: + data_type = self.net_data_type + elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) + if elapsed >= self.timeout: + if gcfg.testing: + msg('') + fetch_delay() + msg_r(f'Fetching data from {self.api_host}...') + if self.has_verbose: + gcfg._util.vmsg('') + data_in = self.get_data_from_network() + msg('done') + if gcfg.testing: + return {} + else: + die(1,self.rate_limit_errmsg(elapsed)) + + if data_type == 'json': + try: + data = json.loads(data_in) + except: + self.json_data_error_msg(data_in) + die(2,'Retrieved data is not valid JSON, exiting') + json_text = data_in + elif data_type == 'python': + data = data_in + json_text = json.dumps(data_in) + + if not data: + if gcfg.cached_data: + die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') + else: + die(2,'Remote host returned no data!') + elif 'error' in data: + die(1,data['error']) + + if gcfg.cached_data: + msg(f'Using cached data from ~/{self.json_fn_rel}') + else: + open(self.json_fn,'w').write(json_text) + msg(f'JSON data cached to ~/{self.json_fn_rel}') + if gcfg.download: + sys.exit(0) + + return self.postprocess_data(data) + + def json_data_error_msg(self,json_text): + pass + + def postprocess_data(self,data): + return data + + @property + def json_fn_rel(self): + return os.path.relpath(self.json_fn,start=homedir) + + class coinpaprika(base): + desc = 'CoinPaprika' + api_host = 'api.coinpaprika.com' + ratelimit = 240 + btc_ratelimit = 10 + net_data_type = 'json' + has_verbose = True + + def rate_limit_errmsg(self,elapsed): + return ( + f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + + ('' if cfg.btc_only else ', or use --cached-data or --btc') + ) + + @property + def api_url(self): + return f'https://{self.api_host}/v1/ticker' + ('/btc-bitcoin' if cfg.btc_only else '') + + @property + def json_fn(self): + return os.path.join( + cfg.cachedir, + 'ticker-btc.json' if cfg.btc_only else 'ticker.json' ) + + @property + def timeout(self): + return 5 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit + + def json_data_error_msg(self,json_text): + tor_captcha_msg = f""" + If you’re using Tor, the API request may have failed due to Captcha protection. + A workaround for this issue is to retrieve the JSON data with a browser from + the following URL: + + {self.api_url} + + and save it to: + + ‘{cfg.cachedir}/ticker.json’ + + Then invoke the program with --cached-data and without --btc + """ + msg(json_text[:1024] + '...') + msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) + + def postprocess_data(self,data): + return [data] if cfg.btc_only else data + + @staticmethod + def parse_asset_id(s,require_label): + sym,label = (*s.split('-',1),None)[:2] + if require_label and not label: + die(1,f'{s!r}: asset label is missing') + return asset_tuple( + symbol = sym.upper(), + id = (s.lower() if label else None), + source = 'cc' ) def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield('') yield(k.upper()) for e in v: - yield(' {:4s} {}'.format(*e.split('-',1))) + out = e.split('-',1) + yield(' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '')) def gen_data(data): """ @@ -59,7 +222,7 @@ def gen_data(data): def dup_sym_errmsg(dup_sym): return ( f'The symbol {dup_sym!r} is shared by the following assets:\n' + - '\n ' + '\n '.join(d['id'] for d in data if d['symbol'] == dup_sym) + + '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + f'instead of {dup_sym!r}' ) @@ -103,13 +266,13 @@ def gen_data(data): wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} - for d in data: + for d in data['cc']: if d['id'] == 'btc-bitcoin': btcusd = Decimal(d['price_usd']) break for k in ('id','symbol'): - for d in data: + for d in data['cc']: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: @@ -149,87 +312,6 @@ def gen_data(data): 'last_updated': None, }) -def get_src_data(curl_cmd): - - tor_captcha_msg = f""" - If you’re using Tor, the API request may have failed due to Captcha protection. - A workaround for this issue is to retrieve the JSON data with a browser from - the following URL: - - {api_url} - - and save it to: - - ‘{cfg.cachedir}/ticker.json’ - - Then invoke the program with --cached-data and without --btc - """ - - def rate_limit_errmsg(timeout,elapsed): - return ( - f'Rate limit exceeded! Retry in {timeout-elapsed} seconds' + - ('' if cfg.btc_only else ', or use --cached-data or --btc') - ) - - if not os.path.exists(cfg.cachedir): - os.makedirs(cfg.cachedir) - - if cfg.btc_only: - fn = os.path.join(cfg.cachedir,'ticker-btc.json') - timeout = 5 if gcfg.test_suite else btc_ratelimit - else: - fn = os.path.join(cfg.cachedir,'ticker.json') - timeout = 5 if gcfg.test_suite else ratelimit - - fn_rel = os.path.relpath(fn,start=homedir) - - if not os.path.exists(fn): - open(fn,'w').write('{}') - - if gcfg.cached_data: - json_text = open(fn).read() - else: - elapsed = int(time.time() - os.stat(fn).st_mtime) - if elapsed >= timeout: - msg_r(f'Fetching data from {api_host}...') - gcfg._util.vmsg('') - try: - cp = run(curl_cmd,check=True,stdout=PIPE) - except CalledProcessError as e: - msg('') - from .Misc import curl_exit_codes - msg(red(curl_exit_codes[e.returncode])) - msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) - from mmgen.exception import MMGenCalledProcessError - raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') - json_text = cp.stdout.decode() - msg('done') - else: - die(1,rate_limit_errmsg(timeout,elapsed)) - - try: - data = json.loads(json_text) - except: - msg(json_text[:1024] + '...') - msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) - die(2,'Retrieved data is not valid JSON, exiting') - - if not data: - if gcfg.cached_data: - die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') - else: - die(2,'Remote host returned no data!') - elif 'error' in data: - die(1,data['error']) - - if gcfg.cached_data: - msg(f'Using cached data from ~/{fn_rel}') - else: - open(fn,'w').write(json_text) - msg(f'JSON data cached to ~/{fn_rel}') - - return data - def main(): def update_sample_file(usr_cfg_file): @@ -243,20 +325,6 @@ def main(): sample_file )) open(sample_file,'w').write(usr_data) - def get_curl_cmd(): - return ([ - 'curl', - '--tr-encoding', - '--compressed', # adds 'Accept-Encoding: gzip' - '--header', 'Accept: application/json', - ] + - (['--proxy', cfg.proxy] if cfg.proxy else []) + - (['--silent'] if not gcfg.verbose else []) + - [api_url + ('/btc-bitcoin' if cfg.btc_only else '')] - ) - - global cfg,cfg_in - try: from importlib.resources import files # Python 3.9 except ImportError: @@ -269,17 +337,24 @@ def main(): die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( os.path.relpath(cfg_in.portfolio_file,start=homedir))) - curl_cmd = get_curl_cmd() + if gcfg.list_ids: + src_ids = ['cc'] + elif gcfg.download: + assert gcfg.download in DataSource.sources, f'{gcfg.download!r}: invalid data source' + src_ids = [gcfg.download] + else: + src_ids = DataSource.sources - if gcfg.print_curl: - Msg(curl_cmd + '\n' + ' '.join(curl_cmd)) + ids = random.sample( list(src_ids), k=len(src_ids) ) # shuffle the ids + + src_data = { k: src_cls[k]().get_data() for k in ids } + + if gcfg.testing: return - src_data = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd) - if gcfg.list_ids: from mmgen.ui import do_pager - do_pager('\n'.join(e['id'] for e in src_data)) + do_pager('\n'.join(e['id'] for e in src_data['cc'])) return global now @@ -304,10 +379,7 @@ def make_cfg(): return tuple(gen()) def parse_asset_id(s,require_label=False): - sym,label = (*s.split('-',1),None)[:2] - if require_label and not label: - die(1,f'{s!r}: asset label is missing') - return asset_tuple( sym.upper(), (s.lower() if label else None) ) + return src_cls['cc'].parse_asset_id(s,require_label) def parse_usr_asset_arg(key,use_cf_file=False): """ @@ -327,7 +399,8 @@ def make_cfg(): None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else Decimal(rate) ), - rate_asset = parse_asset_id(rate_asset) if rate_asset else None ) + rate_asset = parse_asset_id(rate_asset) if rate_asset else None, + source = parsed_id.source ) cl_opt = getattr(gcfg,key) cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] @@ -344,7 +417,8 @@ def make_cfg(): id = parsed_id.id, amount = None if amount is None else Decimal(amount), rate = None, - rate_asset = None ) + rate_asset = None, + source = parsed_id.source ) ss = s.split(':') assert len(ss) in (2,3,4), f'{s}: malformed argument' @@ -417,14 +491,15 @@ def make_cfg(): 'proxy', 'portfolio' ]) - global cfg_in,cfg + global cfg_in,src_cls,cfg + + src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } cmd_args = gcfg._args cfg_in = get_cfg_in() query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset']) - asset_tuple = namedtuple('asset_tuple',['symbol','id']) + asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index b8df683..6553d2a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -12,8 +12,6 @@ mmnode-ticker: Display price information for cryptocurrency and other assets """ -from .Ticker import * - opts_data = { 'sets': [ ('wide', True, 'percent_change', True), @@ -24,7 +22,7 @@ opts_data = { 'text': { 'desc': 'Display prices for cryptocurrency and other assets', 'usage': '[opts] [TRADE_SPECIFIER]', - 'options': f""" + 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -A, --adjust=P Adjust prices by percentage ‘P’. In ‘trading’ mode, @@ -37,6 +35,8 @@ opts_data = { live data from server -D, --cachedir=D Read and write cached JSON data to directory ‘D’ instead of ‘~/{dfl_cachedir}’ +-d, --download=D Retrieve data ‘D’ from source, save to file and exit + (valid options: {ds}) -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -112,7 +112,7 @@ A TRADE_SPECIFIER is a single argument in the format: PROXY NOTE -The remote server used to obtain the price data, {api_host!r}, blocks +The remote server used to obtain the price data, {cc.api_host!r}, blocks Tor behind a Captcha wall, so a Tor proxy cannot be used directly. If you’re concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’ @@ -121,7 +121,7 @@ the script directly on the VPN’ed host with ’proxy’ or --proxy set to the null string. Alternatively, you may download the JSON source data in a Tor-proxied browser -from ‘{api_url}’, save it as ‘ticker.json’ in your +from ‘{cc.api_url}’, save it as ‘ticker.json’ in your configured cache directory, and run the script with the --cached-data option. @@ -130,9 +130,9 @@ configured cache directory, and run the script with the --cached-data option. To protect user privacy, all filtering and processing of data is performed client side so that the remote server does not know which assets are being examined. This means that data for ALL available assets (currently over 4000) -is fetched with each invocation of the script. A rate limit of {L} seconds +is fetched with each invocation of the script. A rate limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the remote server. When the ---btc option is in effect, this limit is reduced to {B} seconds. To bypass the +--btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data. @@ -186,21 +186,20 @@ To add a portfolio, edit the file 'code': { 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), + ds = fmt_dict(DataSource.sources,fmt='equal'), ), 'notes': lambda s: s.format( - assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), - cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), - pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), - api_host = api_host, - api_url = api_url, - L = ratelimit, - B = btc_ratelimit, + assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), + cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), + pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + cc = src_cls['cc'](), ) } } import os +from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config import mmgen_node_tools.Ticker as tck @@ -208,7 +207,7 @@ tck.gcfg = Config( opts_data=opts_data, do_post_init=True ) tck.make_cfg() -from .Ticker import cfg_in +from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls tck.gcfg._post_init() From fd318909c21d602701c2a219bedb1aafa8ffb0d8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 25 Sep 2023 15:53:02 +0000 Subject: [PATCH 03/81] mmnode-ticker: retrieve financial data from Yahoo Finance --- mmgen_node_tools/Ticker.py | 96 +++++++++++++++++++++++++-- mmgen_node_tools/data/ticker-cfg.yaml | 23 ++++--- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 65 +++++++++++------- setup.cfg | 3 +- test/ref/ticker/ticker-cfg.yaml | 16 ++--- test/ref/ticker/ticker-finance.json | 1 + test/ref/ticker/ticker.json | 2 +- test/test_py_d/ts_misc.py | 48 ++++++-------- 9 files changed, 179 insertions(+), 77 deletions(-) create mode 100644 test/ref/ticker/ticker-finance.json diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index cafd407..e6f0650 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -46,6 +46,7 @@ class DataSource: sources = { 'cc': 'coinpaprika', + 'fi': 'yahooquery' } class base: @@ -199,6 +200,69 @@ class DataSource: id = (s.lower() if label else None), source = 'cc' ) + class yahooquery(base): + + desc = 'Yahoo Finance' + api_host = 'finance.yahoo.com' + ratelimit = 30 + net_data_type = 'python' + has_verbose = False + asset_id_pat = r'^\^.*|.*=[xf]$' + + @staticmethod + def get_id(sym,data): + return sym.lower() + + @staticmethod + def conv_data(sym,data,btcusd): + price_usd = Decimal( data['regularMarketPrice']['raw'] ) + return { + 'id': sym, + 'name': data['shortName'], + 'symbol': sym.upper(), + 'price_usd': str(price_usd), + 'price_btc': str(price_usd / btcusd), + 'percent_change_7d': None, + 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, + 'last_updated': data['regularMarketTime'], + } + + def rate_limit_errmsg(self,elapsed): + return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' + + @property + def json_fn(self): + return os.path.join( cfg.cachedir, 'ticker-finance.json' ) + + @property + def timeout(self): + return 5 if gcfg.test_suite else self.ratelimit + + def get_data_from_network(self): + + arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + + kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } } + + if gcfg.test_suite: + kwargs.update({ 'timeout': 1, 'retry': 0 }) + + if gcfg.testing: + Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( + arg, + fmt_dict(kwargs,fmt='kwargs') )) + return + + from yahooquery import Ticker + return Ticker(arg,**kwargs).price + + @staticmethod + def parse_asset_id(s,require_label): + return asset_tuple( + symbol = s.upper(), + id = s.lower(), + source = 'fi' ) + def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield('') @@ -271,6 +335,23 @@ def gen_data(data): btcusd = Decimal(d['price_usd']) break + get_id = src_cls['fi'].get_id + conv_func = src_cls['fi'].conv_data + + for k,v in data['fi'].items(): + id = get_id(k,v) + if wants['id']: + if id in wants['id']: + if id in found['id']: + die(1,dup_sym_errmsg(id)) + yield ( id, conv_func(id,v,btcusd) ) + found['id'].add(id) + wants['id'].remove(id) + if id in usr_rate_assets_want['id']: + rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key + else: + break + for k in ('id','symbol'): for d in data['cc']: if wants[k]: @@ -379,7 +460,7 @@ def make_cfg(): return tuple(gen()) def parse_asset_id(s,require_label=False): - return src_cls['cc'].parse_asset_id(s,require_label) + return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) def parse_usr_asset_arg(key,use_cf_file=False): """ @@ -489,11 +570,13 @@ def make_cfg(): 'add_prec', 'cachedir', 'proxy', + 'proxy2', 'portfolio' ]) global cfg_in,src_cls,cfg src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } + fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args cfg_in = get_cfg_in() @@ -514,6 +597,7 @@ def make_cfg(): proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy + proxy2 = get_proxy('proxy2') cfg = cfg_tuple( rows = create_rows(), @@ -526,6 +610,7 @@ def make_cfg(): add_prec = parse_add_precision(gcfg.add_precision), cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, + proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None ) @@ -541,9 +626,12 @@ def get_cfg_in(): cfg = cfg_data or { 'assets': { 'coin': [ 'btc-bitcoin', 'eth-ethereum', 'xmr-monero' ], - 'commodity': [ 'xau-gold-spot-token', 'xag-silver-spot-token', 'xbr-brent-crude-oil-spot' ], - 'fiat': [ 'gbp-pound-sterling-token', 'eur-euro-token' ], - 'index': [ 'dj30-dow-jones-30-token', 'spx-sp-500', 'ndx-nasdaq-100-token' ], + # gold futures, silver futures, Brent futures + 'commodity': [ 'gc=f', 'si=f', 'bz=f' ], + # Pound Sterling, Euro, Swiss Franc + 'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ], + # Dow Jones Industrials, Nasdaq 100, S&P 500 + 'index': [ '^dji', '^ixic', '^gspc' ], }, 'proxy': 'http://vpn-gw:8118' }, diff --git a/mmgen_node_tools/data/ticker-cfg.yaml b/mmgen_node_tools/data/ticker-cfg.yaml index 6589bfc..6344445 100644 --- a/mmgen_node_tools/data/ticker-cfg.yaml +++ b/mmgen_node_tools/data/ticker-cfg.yaml @@ -3,10 +3,16 @@ ### See the curl manpage for supported --proxy parameters ### For a direct connection, leave the right-hand side blank proxy: http://vpn-gw:8118 +# proxy2: http://gw2:8118 ### Override the default cache directory (~/.cache/mmgen-node-tools): cachedir: +### Additional asset columns: +# add_columns: +# - cnhusd=x # Yuan +# - 6j=f # Yen futures + ### Asset rows ### Asset labels are arbitrary strings. Use as many or few as you wish. ### Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs. @@ -19,13 +25,14 @@ assets: - ada-cardano - bnb-binance-coin commodity: - - xau-gold-spot-token - - xag-silver-spot-token - - xbr-brent-crude-oil-spot + - gc=f # gold futures + - si=f # silver futures + - bz=f # Brent futures fiat: - - gbp-pound-sterling-token - - eur-euro-token + - gbpusd=x # Pound Sterling + - eurusd=x # Euro + - chfusd=x # Swiss Franc index: - - dj30-dow-jones-30-token - - spx-sp-500 - - ndx-nasdaq-100-token + - ^dji # Dow Jones Industrials + - ^ixic # Nasdaq 100 + - ^gspc # S&P 500 diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index b473ce1..0c84a84 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev0 +3.2.dev1 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 6553d2a..f329e03 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -47,7 +47,7 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. --t, --testing Print command to be executed to stdout and exit +-t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column -v, --verbose Be more verbose @@ -55,6 +55,8 @@ opts_data = { -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to completely disable or ‘none’ to allow override from environment. Consult the curl manpage for --proxy usage. +-X, --proxy2=P Alternate proxy for non-crypto financial data. Defaults + to value of --proxy """, 'notes': """ @@ -69,10 +71,15 @@ price to the spot price. ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids) consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the -symbol is ambiguous, the full ID must be used. Examples: +symbol is ambiguous, the full ID must be used. For Yahoo Finance assets +the symbol and ID are identical: - chf - specify asset by symbol - chf-swiss-franc-token - same as above, but use full ID instead of symbol +Examples: + + ltc - specify asset by symbol + ltc-litecoin - same as above, but use full ID instead of symbol + ^dji - Dow Jones Industrial Average (Yahoo) + gc=f - gold futures (Yahoo) ASSET SPECIFIERS have the following format: @@ -89,7 +96,8 @@ postfixed with the letter ‘r’, its meaning is reversed, i.e. interpreted as inr:0.01257r - same as above, but use reverse rate (INR/USD) inr-indian-rupee:79.5 - same as first example, but add an arbitrary label omr-omani-rial:2.59r - Omani Rial is pegged to the Dollar at 2.59 USD - bgn-bg-lev:0.5113r:eur - Bulgarian Lev is pegged to the Euro at 0.5113 EUR + bgn-bulgarian-lev:0.5113r:eurusd=x + - Bulgarian Lev is pegged to the Euro at 0.5113 EUR A TRADE_SPECIFIER is a single argument in the format: @@ -99,8 +107,8 @@ A TRADE_SPECIFIER is a single argument in the format: xmr:17.34 - price of 17.34 XMR in all configured assets xmr-monero:17.34 - same as above, but with full ID - xmr:17.34:eur - price of 17.34 XMR in EUR only - xmr:17.34:eur:2800 - commission on an offer of 17.34 XMR for 2800 EUR + xmr:17.34:eurusd=x - price of 17.34 XMR in EUR only + xmr:17.34:eurusd=x:2800 - commission on an offer of 17.34 XMR for 2800 EUR TO_AMOUNT, if included, is used to calculate the percentage difference or commission on an offer compared to the spot price. @@ -112,28 +120,34 @@ A TRADE_SPECIFIER is a single argument in the format: PROXY NOTE -The remote server used to obtain the price data, {cc.api_host!r}, blocks -Tor behind a Captcha wall, so a Tor proxy cannot be used directly. If you’re -concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then -set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’ -option in the config file or --proxy on the command line accordingly. Or run -the script directly on the VPN’ed host with ’proxy’ or --proxy set to the -null string. +The remote server used to obtain the crypto price data, {cc.api_host}, +blocks Tor behind a Captcha wall, so a Tor proxy cannot be used directly. +If you’re concerned about privacy, connect via a VPN, or better yet, VPN over +Tor. Then set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the +‘proxy’ option in the config file or --proxy on the command line accordingly. +Or run the script directly on the VPN’ed host with ’proxy’ or --proxy set to +the null string. Alternatively, you may download the JSON source data in a Tor-proxied browser -from ‘{cc.api_url}’, save it as ‘ticker.json’ in your -configured cache directory, and run the script with the --cached-data option. +from {cc.api_url}, save it as ‘ticker.json’ in your +configured cache directory and run the script with the --cached-data option. + +Financial data is obtained from {fi.desc}, which currently allows Tor. RATE LIMITING NOTE -To protect user privacy, all filtering and processing of data is performed -client side so that the remote server does not know which assets are being -examined. This means that data for ALL available assets (currently over 4000) -is fetched with each invocation of the script. A rate limit of {cc.ratelimit} seconds -between calls is thus imposed to prevent abuse of the remote server. When the ---btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the -rate limit entirely, use --cached-data. +To protect user privacy, all filtering and processing of cryptocurrency data +is performed client side so that the remote server does not know which assets +are being examined. This means that data for ALL available crypto assets +(currently over 8000) is fetched with each invocation of the script. A rate +limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the +remote server. When the --btc option is in effect, this limit is reduced to +{cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data. + +Note that financial data obtained from {fi.api_host} is filtered in the +request, which has privacy implications. The rate limit for financial data +is {fi.ratelimit} seconds. EXAMPLES @@ -146,10 +160,10 @@ $ mmnode-ticker --btc # Wide display, add EUR and OMR columns, OMR/USD rate, extra precision and # proxy: -$ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 +$ mmnode-ticker -w -c eurusd=x,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 # Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate: -$ mmnode-ticker -wE -c eur,bgn-bulgarian-lev:0.5113r:eur +$ mmnode-ticker -wE -c eurusd=x,bgn-bulgarian-lev:0.5113r:eurusd=x # Wide display, use cached data from previous network query, show portfolio # (see above), pipe output to pager, add DOGE row: @@ -193,6 +207,7 @@ To add a portfolio, edit the file cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), cc = src_cls['cc'](), + fi = src_cls['fi'](), ) } } diff --git a/setup.cfg b/setup.cfg index 0cdbed1..9539a29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,8 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=13.3.dev44 + mmgen>=14.0.dev2 + yahooquery packages = mmgen_node_tools diff --git a/test/ref/ticker/ticker-cfg.yaml b/test/ref/ticker/ticker-cfg.yaml index 504a7fb..4b40a43 100644 --- a/test/ref/ticker/ticker-cfg.yaml +++ b/test/ref/ticker/ticker-cfg.yaml @@ -12,13 +12,13 @@ assets: - ada-cardano - algo-algorand commodity: - - xau-gold-spot-token - - xag-silver-spot-token - - xbr-brent-crude-oil-spot + - gc=f # gold futures + - si=f # silver futures + - bz=f # Brent futures fiat: - - chf-swiss-franc-token - - eur-euro-token + - chfusd=x # Swiss Franc + - eurusd=x # Euro index: - - dj30-dow-jones-30-token - - spx-sp-500 - - ndx-nasdaq-100-token + - ^dji # Dow Jones Industrials + - ^ixic # Nasdaq 100 + - ^gspc # S&P 500 diff --git a/test/ref/ticker/ticker-finance.json b/test/ref/ticker/ticker-finance.json new file mode 100644 index 0000000..ac0af29 --- /dev/null +++ b/test/ref/ticker/ticker-finance.json @@ -0,0 +1 @@ +{"GC=F":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0015419408,"fmt":"-0.15%"},"regularMarketChange":{"raw":-3,"fmt":"-3.00"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":1942.6,"fmt":"1,942.60"},"regularMarketDayHigh":{"raw":1946.8,"fmt":"1,946.80"},"regularMarketDayLow":{"raw":1940.1,"fmt":"1,940.10"},"regularMarketVolume":{"raw":39539,"fmt":"39.54k","longFmt":"39,539.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1945.6,"fmt":"1,945.60"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1944.7,"fmt":"1,944.70"},"strikePrice":{},"openInterest":{},"exchange":"CMX","exchangeName":"COMEX","exchangeDataDelayedBy":10,"marketState":"REGULAR","quoteType":"FUTURE","symbol":"GC=F","underlyingSymbol":null,"shortName":"Gold Dec 23","longName":null,"currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"SI=F":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0014259518,"fmt":"-0.14%"},"regularMarketChange":{"raw":-0.034000397,"fmt":"-0.03"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":23.81,"fmt":"23.81"},"regularMarketDayHigh":{"raw":23.925,"fmt":"23.92"},"regularMarketDayLow":{"raw":23.67,"fmt":"23.67"},"regularMarketVolume":{"raw":11819,"fmt":"11.82k","longFmt":"11,819.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":23.844,"fmt":"23.84"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":23.81,"fmt":"23.81"},"strikePrice":{},"openInterest":{},"exchange":"CMX","exchangeName":"COMEX","exchangeDataDelayedBy":10,"marketState":"REGULAR","quoteType":"FUTURE","symbol":"SI=F","underlyingSymbol":null,"shortName":"Silver Dec 23","longName":null,"currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"BZ=F":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":0.004610275,"fmt":"0.46%"},"regularMarketChange":{"raw":0.4300003,"fmt":"0.43"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":93.7,"fmt":"93.70"},"regularMarketDayHigh":{"raw":94.25,"fmt":"94.25"},"regularMarketDayLow":{"raw":93.24,"fmt":"93.24"},"regularMarketVolume":{"raw":2853,"fmt":"2.85k","longFmt":"2,853.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":93.27,"fmt":"93.27"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":93.75,"fmt":"93.75"},"strikePrice":{},"openInterest":{},"exchange":"NYM","exchangeName":"NY Mercantile","exchangeDataDelayedBy":10,"marketState":"REGULAR","quoteType":"FUTURE","symbol":"BZ=F","underlyingSymbol":null,"shortName":"Brent Crude Oil Last Day Financ","longName":null,"currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"GBPUSD=X":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.00045271934,"fmt":"-0.0453%"},"regularMarketChange":{"raw":-0.0005540848,"fmt":"-0.0006"},"regularMarketTime":1659385900,"priceHint":{"raw":4,"fmt":"4","longFmt":"4"},"regularMarketPrice":{"raw":1.223481,"fmt":"1.2235"},"regularMarketDayHigh":{"raw":1.2263919,"fmt":"1.2264"},"regularMarketDayLow":{"raw":1.2213293,"fmt":"1.2213"},"regularMarketVolume":{"raw":0,"fmt":null,"longFmt":"0.0000"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1.2240351,"fmt":"1.2240"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1.2240951,"fmt":"1.2241"},"strikePrice":{},"openInterest":{},"exchange":"CCY","exchangeName":"CCY","exchangeDataDelayedBy":0,"marketState":"REGULAR","quoteType":"CURRENCY","symbol":"GBPUSD=X","underlyingSymbol":null,"shortName":"GBP/USD","longName":"GBP/USD","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"EURUSD=X":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0005321096,"fmt":"-0.0532%"},"regularMarketChange":{"raw":-0.00056660175,"fmt":"-0.0006"},"regularMarketTime":1659385900,"priceHint":{"raw":4,"fmt":"4","longFmt":"4"},"regularMarketPrice":{"raw":1.0641694,"fmt":"1.0642"},"regularMarketDayHigh":{"raw":1.0658708,"fmt":"1.0659"},"regularMarketDayLow":{"raw":1.0626993,"fmt":"1.0627"},"regularMarketVolume":{"raw":0,"fmt":null,"longFmt":"0.0000"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1.064736,"fmt":"1.0647"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1.064736,"fmt":"1.0647"},"strikePrice":{},"openInterest":{},"exchange":"CCY","exchangeName":"CCY","exchangeDataDelayedBy":0,"marketState":"REGULAR","quoteType":"CURRENCY","symbol":"EURUSD=X","underlyingSymbol":null,"shortName":"EUR/USD","longName":"EUR/USD","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"CHFUSD=X":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0028495365,"fmt":"-0.2850%"},"regularMarketChange":{"raw":-0.0031440258,"fmt":"-0.0031"},"regularMarketTime":1659385900,"priceHint":{"raw":4,"fmt":"4","longFmt":"4"},"regularMarketPrice":{"raw":1.1002069,"fmt":"1.1002"},"regularMarketDayHigh":{"raw":1.1046062,"fmt":"1.1046"},"regularMarketDayLow":{"raw":1.0982735,"fmt":"1.0983"},"regularMarketVolume":{"raw":0,"fmt":null,"longFmt":"0.0000"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":1.1033509,"fmt":"1.1034"},"regularMarketSource":"DELAYED","regularMarketOpen":{"raw":1.1043622,"fmt":"1.1044"},"strikePrice":{},"openInterest":{},"exchange":"CCY","exchangeName":"CCY","exchangeDataDelayedBy":0,"marketState":"REGULAR","quoteType":"CURRENCY","symbol":"CHFUSD=X","underlyingSymbol":null,"shortName":"CHF/USD","longName":"CHF/USD","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"^DJI":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0031276005,"fmt":"-0.31%"},"regularMarketChange":{"raw":-106.55859,"fmt":"-106.56"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":33963.84,"fmt":"33,963.84"},"regularMarketDayHigh":{"raw":34156.15,"fmt":"34,156.15"},"regularMarketDayLow":{"raw":33947.24,"fmt":"33,947.24"},"regularMarketVolume":{"raw":271273859,"fmt":"271.27M","longFmt":"271,273,859.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":34070.4,"fmt":"34,070.40"},"regularMarketSource":"FREE_REALTIME","regularMarketOpen":{"raw":34077.08,"fmt":"34,077.08"},"strikePrice":{},"openInterest":{},"exchange":"DJI","exchangeName":"DJI","exchangeDataDelayedBy":0,"marketState":"PRE","quoteType":"INDEX","symbol":"^DJI","underlyingSymbol":null,"shortName":"Dow Jones Industrial Average","longName":"Dow Jones Industrial Average","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"^IXIC":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.00092206284,"fmt":"-0.09%"},"regularMarketChange":{"raw":-12.193359,"fmt":"-12.19"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":13211.807,"fmt":"13,211.81"},"regularMarketDayHigh":{"raw":13353.22,"fmt":"13,353.22"},"regularMarketDayLow":{"raw":13200.639,"fmt":"13,200.64"},"regularMarketVolume":{"raw":3887623000,"fmt":"3.89B","longFmt":"3,887,623,000.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":13224,"fmt":"13,224.00"},"regularMarketSource":"FREE_REALTIME","regularMarketOpen":{"raw":13287.171,"fmt":"13,287.17"},"strikePrice":{},"openInterest":{},"exchange":"NIM","exchangeName":"Nasdaq GIDS","exchangeDataDelayedBy":0,"marketState":"PRE","quoteType":"INDEX","symbol":"^IXIC","underlyingSymbol":null,"shortName":"NASDAQ Composite","longName":"NASDAQ Composite","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}},"^GSPC":{"maxAge":1,"preMarketChange":{},"preMarketPrice":{},"postMarketChange":{},"postMarketPrice":{},"regularMarketChangePercent":{"raw":-0.0022955984,"fmt":"-0.23%"},"regularMarketChange":{"raw":-9.939941,"fmt":"-9.94"},"regularMarketTime":1659385900,"priceHint":{"raw":2,"fmt":"2","longFmt":"2"},"regularMarketPrice":{"raw":4320.06,"fmt":"4,320.06"},"regularMarketDayHigh":{"raw":4357.4,"fmt":"4,357.40"},"regularMarketDayLow":{"raw":4316.49,"fmt":"4,316.49"},"regularMarketVolume":{"raw":2135953000,"fmt":"2.14B","longFmt":"2,135,953,000.00"},"averageDailyVolume10Day":{},"averageDailyVolume3Month":{},"regularMarketPreviousClose":{"raw":4330,"fmt":"4,330.00"},"regularMarketSource":"FREE_REALTIME","regularMarketOpen":{"raw":4341.74,"fmt":"4,341.74"},"strikePrice":{},"openInterest":{},"exchange":"SNP","exchangeName":"SNP","exchangeDataDelayedBy":0,"marketState":"PRE","quoteType":"INDEX","symbol":"^GSPC","underlyingSymbol":null,"shortName":"S&P 500","longName":"S&P 500","currency":"USD","quoteSourceName":"Delayed Quote","currencySymbol":"$","fromCurrency":null,"toCurrency":null,"lastMarket":null,"volume24Hr":{},"volumeAllCurrencies":{},"circulatingSupply":{},"marketCap":{}}} diff --git a/test/ref/ticker/ticker.json b/test/ref/ticker/ticker.json index 2484e6d..880a666 100644 --- a/test/ref/ticker/ticker.json +++ b/test/ref/ticker/ticker.json @@ -1 +1 @@ -[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"},{"id":"xau-gold-spot-token","name":"Gold Spot Token","symbol":"XAU","rank":"2634","price_usd":"1767.0700000000002","price_btc":"0.07608887785567188","volume_24h_usd":"11881071.852000002","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.33","percent_change_24h":"-0.19","percent_change_7d":"2.89","last_updated":"1659464759"},{"id":"xag-silver-spot-token","name":"Silver Spot Token","symbol":"XAG","rank":"4753","price_usd":"20.0265","price_btc":"0.0008623279849562342","volume_24h_usd":"445379.34674999997","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.61","percent_change_24h":"-1.54","percent_change_7d":"7.49","last_updated":"1659464759"},{"id":"xbr-brent-crude-oil-spot","name":"Brent Crude Oil Spot","symbol":"XBR","rank":"2616","price_usd":"100.39000000000001","price_btc":"0.0043227277062770015","volume_24h_usd":"4527788.105237802","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.89","percent_change_24h":"0.87","percent_change_7d":"1.02","last_updated":"1659464759"},{"id":"chf-swiss-franc-token","name":"Swiss Franc Token","symbol":"CHF","rank":"3234","price_usd":"1.0453383927173951","price_btc":"0.00004501158713651311","volume_24h_usd":"360978.80080814153","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.08","percent_change_24h":"-0.66","percent_change_7d":"0.65","last_updated":"1659464759"},{"id":"eur-euro-token","name":"Euro Token","symbol":"EUR","rank":"5116","price_usd":"1.0185784743417672","price_btc":"0.00004385932256255119","volume_24h_usd":"40587878.087926075","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"0.05","percent_change_24h":"-0.82","percent_change_7d":"0.6","last_updated":"1659464759"},{"id":"gbp-pound-sterling-token","name":"Pound Sterling Token","symbol":"GBP","rank":"3424","price_usd":"1.2181792934946465","price_btc":"0.00005245400321946659","volume_24h_usd":"5428622.322314565","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.61","percent_change_7d":"1.29","last_updated":"1659464759"},{"id":"dj30-dow-jones-30-token","name":"Dow Jones 30 Token","symbol":"DJ30","rank":"3192","price_usd":"32546","price_btc":"1.401409462381624","volume_24h_usd":"293564.92","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.1","percent_change_24h":"-0.91","percent_change_7d":"2.5","last_updated":"1659464759"},{"id":"spx-sp-500","name":"S&P 500 Token","symbol":"SPX","rank":"3342","price_usd":"4110.6","price_btc":"0.17699974608449287","volume_24h_usd":"780397.41","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"-0.27","percent_change_7d":"4.87","last_updated":"1659464759"},{"id":"ndx-nasdaq-100-token","name":"NASDAQ 100 Token","symbol":"NDX","rank":"5097","price_usd":"12948.199999999999","price_btc":"0.5575410188904857","volume_24h_usd":"11221816.494","market_cap_usd":"","circulating_supply":"","total_supply":"","max_supply":"","percent_change_1h":"0.08","percent_change_24h":"0.04","percent_change_7d":"7.21","last_updated":"1659464759"}] +[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"}] diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index c1e925d..28a9e8d 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -87,7 +87,6 @@ class TestSuiteScripts(TestSuiteBase): ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), ('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'), - ('ticker18', 'ticker [--cached-data --wide --elapsed -c eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'), ) } @@ -102,6 +101,7 @@ class TestSuiteScripts(TestSuiteBase): def ticker_setup(self): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) + shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' @@ -123,8 +123,8 @@ class TestSuiteScripts(TestSuiteBase): if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - t.expect('proxy host could not be resolved') - t.req_exit_val = 3 + ret = t.expect(['proxy host could not be resolved','ProxyError']) + t.req_exit_val = 3 if ret == 0 else 1 return t def ticker3(self): @@ -138,15 +138,15 @@ class TestSuiteScripts(TestSuiteBase): def ticker4(self): return self.ticker( - ['--wide','--add-columns=eur,inr-indian-rupee:79.5'], + ['--wide','--add-columns=eurusd=x,inr-indian-rupee:79.5'], [ - r'EUR \(EURO TOKEN\) = 1.0186 USD ' + + r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' + r'INR \(INDIAN RUPEE\) = 0.012579 USD', - 'USD EUR INR BTC CHG_7d CHG_24h UPDATED', + 'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED', 'BITCOIN', - r'ETHEREUM 1,659.66 1,629.3906 131,943.14 0.07146397 \+21.42 \+1.82', - r'MONERO 158.97 156.0734 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', - r'INDIAN RUPEE 0.01 0.0123 1.00 0.00000054 -- --', + r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07146397 \+21.42 \+1.82', + r'MONERO 158.97 149.3870 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', + r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --', ]) def ticker5(self): @@ -213,7 +213,7 @@ class TestSuiteScripts(TestSuiteBase): 'SPOT PRICE', 'BTC 0.11783441', 'XMR 17.23400000', - 'XAU','NDX', + 'GC=F',r'\^IXIC', ]) def ticker11(self): @@ -277,32 +277,22 @@ class TestSuiteScripts(TestSuiteBase): def ticker16(self): return self.ticker( - ['--wide','--elapsed','-c','eur,omr-omani-rial:2.59r'], + ['--wide','--elapsed','-c','eurusd=x,omr-omani-rial:2.59r'], [ - r'EUR \(EURO TOKEN\) = 1.0186 USD ' + + r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' + r'OMR \(OMANI RIAL\) = 2.5900 USD', - 'USD EUR OMR BTC CHG_7d CHG_24h UPDATED', - r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', - 'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- --' + 'USD EURUSD=X OMR BTC CHG_7d CHG_24h UPDATED', + r'BITCOIN 23,250.77 21,848.7527 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago', + 'OMANI RIAL 2.59 2.4338 1.0000 0.00011139 -- -- --' ]) def ticker17(self): # BGN pegged at 0.5113 EUR return self.ticker( - ['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eur'], + ['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eurusd=x'], [ - r'BGN \(BULGARIAN LEV\) = 0.52080 USD', + r'BGN \(BULGARIAN LEV\) = 0.54411 USD', 'USD BGN BTC CHG_7d CHG_24h UPDATED', - 'BITCOIN 23,250.77 44,644.414 1.00000000', - 'BULGARIAN LEV 0.52 1.000 0.00002240', - ]) - - def ticker18(self): - return self.ticker( - ['--wide','--elapsed','-c','eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'], - [ - r'BGN \(BULGARIAN LEV\) = 0.52080 USD', - 'USD EUR BGN BTC CHG_7d CHG_24h UPDATED', - 'BITCOIN 23,250.77 22,826.6890 44,644.414 1.00000000', - 'BULGARIAN LEV 0.52 0.5113 1.000 0.00002240', + 'BITCOIN 23,250.77 42,731.767 1.00000000', + 'BULGARIAN LEV 0.54 1.000 0.00002340', ]) From df620f923f472ef9d03fd3673395511e8b473c8b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 3 Oct 2023 14:35:57 +0000 Subject: [PATCH 04/81] minor fixes and cleanups --- mmgen_node_tools/Sound.py | 5 ++++- mmgen_node_tools/Ticker.py | 28 +++++++++++++++------------- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 24 ++++++++++++------------ mmgen_node_tools/main_txfind.py | 2 +- setup.cfg | 2 +- test/test_py_d/ts_main.py | 7 +++---- test/test_py_d/ts_misc.py | 8 ++++---- test/test_py_d/ts_regtest.py | 8 ++++---- test/unit_tests_d/ut_BlocksInfo.py | 4 +--- 10 files changed, 46 insertions(+), 44 deletions(-) diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index 271253c..88d3b02 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -20,7 +20,10 @@ mmgen_node_tools.Sound: audio-related functions for MMGen node tools """ import sys,os,time -from mmgen_node_tools.Util import * + +from mmgen.util import die + +from mmgen_node_tools.Util import do_system _alsa_config_file = '/tmp/alsa-config-' + os.path.basename(sys.argv[0]) _dvols = { 'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190 } diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index e6f0650..97b5dc5 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import * +from mmgen.color import red,yellow,green,blue,orange,gray from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen homedir = os.getenv('HOME') @@ -37,8 +37,9 @@ def fetch_delay(fetched_data=[]): if not gcfg.testing: if fetched_data: delay = 1 + random.randrange(1,5000) / 1000 - msg(f'Waiting {delay:.3f} seconds...') + msg_r(f'Waiting {delay:.3f} seconds...') time.sleep(delay) + msg('') else: fetched_data.append(None) @@ -265,11 +266,11 @@ class DataSource: def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): - yield('') - yield(k.upper()) + yield '' + yield k.upper() for e in v: out = e.split('-',1) - yield(' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '')) + yield ' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '') def gen_data(data): """ @@ -421,7 +422,8 @@ def main(): if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: - assert gcfg.download in DataSource.sources, f'{gcfg.download!r}: invalid data source' + if not gcfg.download in DataSource.sources: + die(1,f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: src_ids = DataSource.sources @@ -449,19 +451,22 @@ def main(): def make_cfg(): + query_tuple = namedtuple('query',['asset','to_asset']) + asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) + + def parse_asset_id(s,require_label=False): + return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) + def get_rows_from_cfg(add_data=None): def gen(): for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): - yield(k) + yield k if add_data and k in add_data: v += tuple(add_data[k]) for e in v: yield parse_asset_id(e,require_label=True) return tuple(gen()) - def parse_asset_id(s,require_label=False): - return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) - def parse_usr_asset_arg(key,use_cf_file=False): """ asset_id[:rate[:rate_asset]] @@ -581,9 +586,6 @@ def make_cfg(): cmd_args = gcfg._args cfg_in = get_cfg_in() - query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) - usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0c84a84..4b653e3 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev1 +3.2.dev2 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 72692fc..7bee773 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -145,25 +145,25 @@ def create_data(coin_amt,mempool): def gen_header(host,mempool,blockcount): - yield(fmt(f""" + yield fmt(f""" Mempool Fee Structure Date: {make_timestr()} UTC Host: {host} Network: {proto.coin.upper()} {proto.network.upper()} Block: {blockcount} TX count: {len(mempool)} - """)).strip() + """).strip() if cfg.show_empty: - yield('Displaying all fee brackets') + yield 'Displaying all fee brackets' elif cfg.ignore_below: - yield('Ignoring fee brackets with less than {:,} bytes ({})'.format( - ignore_below, - int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), - )) + yield 'Ignoring fee brackets with less than {:,} bytes ({})'.format( + ignore_below, + int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), + ) if cfg.include_current: - yield('Including transactions in current fee bracket in Total MB amounts') + yield 'Including transactions in current fee bracket in Total MB amounts' def fmt_mb(n): return int2bytespec(n,'MB',f'0.{precision}',print_sym=False) @@ -187,17 +187,17 @@ def gen_body(data): for i in data: if not i.skip: cum_bytes = i.tx_bytes_cum + i.tx_bytes if cfg.include_current else i.tx_bytes_cum - yield(fs.format( + yield fs.format( a = '{}-{}'.format(i.bottom,i.top) if cfg.ranges else i.top, b = fmt_mb(i.tx_bytes), c = fmt_mb(cum_bytes), - d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) )) + d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) ) - yield(fs.format( + yield fs.format( a = 'TOTAL', b = '', c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes if data else 0), - d = '' )) + d = '' ) async def main(): diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index cc8df14..57b691a 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -21,7 +21,7 @@ mmnode-txfind: Find a transaction in the blockchain or mempool """ from mmgen.cfg import Config -from mmgen.util import msg,Msg,die +from mmgen.util import msg,Msg,die,is_hex_str opts_data = { 'text': { diff --git a/setup.cfg b/setup.cfg index 9539a29..aeebff5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=14.0.dev2 + mmgen>=14.0.dev6 yahooquery packages = diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index 162e3fb..903981c 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -12,11 +12,10 @@ test_py_d.ts_main: Basic operations tests for the test.py test suite """ -import time +import sys,time -from ..include.common import * -from .common import * -from .ts_base import * +from ..include.common import cfg +from .ts_base import TestSuiteBase class TestSuiteMain(TestSuiteBase): 'basic operations with fake RPC data' diff --git a/test/test_py_d/ts_misc.py b/test/test_py_d/ts_misc.py index 28a9e8d..06674ed 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/test_py_d/ts_misc.py @@ -12,10 +12,10 @@ test.test_py_d.ts_misc: Miscellaneous test groups for the test.py test suite """ -import shutil -from ..include.common import * -from .common import * -from .ts_base import * +import os,shutil + +from ..include.common import cfg +from .ts_base import TestSuiteBase refdir = os.path.join('test','ref','ticker') diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index f8859b8..2ed45b4 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -13,13 +13,13 @@ test.test_py_d.ts_regtest: Regtest tests for the test.py test suite """ import os -from mmgen.util import die,gmsg + +from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest -from ..include.common import * -from .common import * -from .ts_base import * +from ..include.common import cfg,imsg,stop_test_daemons,joinpath +from .ts_base import TestSuiteBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/unit_tests_d/ut_BlocksInfo.py index e65e314..b301a66 100755 --- a/test/unit_tests_d/ut_BlocksInfo.py +++ b/test/unit_tests_d/ut_BlocksInfo.py @@ -3,11 +3,9 @@ test.unit_tests_d.nt_BlocksInfo: BlocksInfo unit test for the MMGen Node Tools suite """ -from mmgen.common import * -from mmgen.exception import * from mmgen_node_tools.BlocksInfo import BlocksInfo -from ..include.common import cfg,vmsg +from ..include.common import vmsg tip = 50000 range_vecs = ( From 666b27c04200ff4312b04a857af0ab9ea88c2518 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 13 Oct 2023 09:50:15 +0000 Subject: [PATCH 05/81] test rename: test.py -> cmdtest.py --- test/{test_py_d => cmdtest_py_d}/cfg.py | 10 +-- .../ts_main.py => cmdtest_py_d/ct_main.py} | 6 +- .../ts_misc.py => cmdtest_py_d/ct_misc.py} | 8 +-- .../ct_regtest.py} | 8 +-- test/init.sh | 65 ++++++++++++------- test/test-release.d/cfg.sh | 12 ++-- 6 files changed, 64 insertions(+), 45 deletions(-) rename test/{test_py_d => cmdtest_py_d}/cfg.py (67%) rename test/{test_py_d/ts_main.py => cmdtest_py_d/ct_main.py} (93%) rename test/{test_py_d/ts_misc.py => cmdtest_py_d/ct_misc.py} (97%) rename test/{test_py_d/ts_regtest.py => cmdtest_py_d/ct_regtest.py} (98%) diff --git a/test/test_py_d/cfg.py b/test/cmdtest_py_d/cfg.py similarity index 67% rename from test/test_py_d/cfg.py rename to test/cmdtest_py_d/cfg.py index c2c72e3..5aacbf9 100755 --- a/test/test_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -9,16 +9,16 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.test_py_d.cfg: configuration data for test.py +test.cmdtest_py_d.cfg: configuration data for cmdtest.py """ import os cmd_groups_dfl = { - 'main': ('TestSuiteMain',{}), - 'helpscreens': ('TestSuiteHelp',{'modname':'misc','full_data':True}), - 'scripts': ('TestSuiteScripts',{'modname':'misc'}), - 'regtest': ('TestSuiteRegtest',{}), + 'main': ('CmdTestMain',{}), + 'helpscreens': ('CmdTestHelp',{'modname':'misc','full_data':True}), + 'scripts': ('CmdTestScripts',{'modname':'misc'}), + 'regtest': ('CmdTestRegtest',{}), } cmd_groups_extra = {} diff --git a/test/test_py_d/ts_main.py b/test/cmdtest_py_d/ct_main.py similarity index 93% rename from test/test_py_d/ts_main.py rename to test/cmdtest_py_d/ct_main.py index 903981c..ad1ce41 100755 --- a/test/test_py_d/ts_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -9,15 +9,15 @@ # https://gitlab.com/mmgen/mmgen """ -test_py_d.ts_main: Basic operations tests for the test.py test suite +cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite """ import sys,time from ..include.common import cfg -from .ts_base import TestSuiteBase +from .ct_base import CmdTestBase -class TestSuiteMain(TestSuiteBase): +class CmdTestMain(CmdTestBase): 'basic operations with fake RPC data' tmpdir_nums = [3] networks = ('btc',) # fake data, so test peerblocks for BTC mainnet only diff --git a/test/test_py_d/ts_misc.py b/test/cmdtest_py_d/ct_misc.py similarity index 97% rename from test/test_py_d/ts_misc.py rename to test/cmdtest_py_d/ct_misc.py index 06674ed..966fb91 100755 --- a/test/test_py_d/ts_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -9,17 +9,17 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.test_py_d.ts_misc: Miscellaneous test groups for the test.py test suite +test.cmdtest_py_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite """ import os,shutil from ..include.common import cfg -from .ts_base import TestSuiteBase +from .ct_base import CmdTestBase refdir = os.path.join('test','ref','ticker') -class TestSuiteHelp(TestSuiteBase): +class CmdTestHelp(CmdTestBase): 'help, info and usage screens' networks = ('btc','ltc','bch') tmpdir_nums = [] @@ -52,7 +52,7 @@ class TestSuiteHelp(TestSuiteBase): def longhelpscreens(self): return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') -class TestSuiteScripts(TestSuiteBase): +class CmdTestScripts(CmdTestBase): 'scripts not requiring a coin daemon' networks = ('btc',) tmpdir_nums = [2] diff --git a/test/test_py_d/ts_regtest.py b/test/cmdtest_py_d/ct_regtest.py similarity index 98% rename from test/test_py_d/ts_regtest.py rename to test/cmdtest_py_d/ct_regtest.py index 2ed45b4..a03e3ff 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.test_py_d.ts_regtest: Regtest tests for the test.py test suite +test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import os @@ -19,7 +19,7 @@ from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest from ..include.common import cfg,imsg,stop_test_daemons,joinpath -from .ts_base import TestSuiteBase +from .ct_base import CmdTestBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] @@ -31,7 +31,7 @@ def gen_addrs(proto,network,keys): tool.addrtype = proto.mmtypes[-1] return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] -class TestSuiteRegtest(TestSuiteBase): +class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' networks = ('btc','ltc','bch') passthru_opts = ('coin',) @@ -104,7 +104,7 @@ class TestSuiteRegtest(TestSuiteBase): } def __init__(self,trunner,cfgs,spawn): - TestSuiteBase.__init__(self,trunner,cfgs,spawn) + CmdTestBase.__init__(self,trunner,cfgs,spawn) if trunner == None: return if self.proto.testnet: diff --git a/test/init.sh b/test/init.sh index a8d03a9..913ac9c 100755 --- a/test/init.sh +++ b/test/init.sh @@ -45,37 +45,56 @@ build_mmgen_extmod() { } create_dir_links() { - for target in 'mmgen' 'scripts'; do - src="$mm_repo/$target" - if [ -e $target ]; then - [ $(realpath --relative-to=. $target) == $src ] || die "'$target' does not point to '$src'" + for link_name in 'mmgen' 'scripts'; do + target="$mm_repo/$link_name" + if [ -e $link_name ]; then + [ $(realpath --relative-to=. $link_name) == $target ] || die "'$link_name' does not point to '$target'" else - echo "Creating symlink: $target" - ln -s $src + echo "Creating symlink: $link_name" + ln -s $target fi done } create_test_links() { - sources=' - test/include - test/overlay/__init__.py - test/overlay/fakemods/mmgen - test/__init__.py - test/test.py - test/unit_tests.py - test/test-release.sh - test/test_py_d/common.py - test/test_py_d/ts_base.py - cmds/mmgen-regtest + paths=' + test/include symbolic + test/overlay/__init__.py symbolic + test/overlay/fakemods/mmgen symbolic + test/__init__.py symbolic + test/cmdtest.py hard + test/unit_tests.py hard + test/test-release.sh symbolic + test/cmdtest_py_d/common.py symbolic + test/cmdtest_py_d/ct_base.py symbolic + cmds/mmgen-regtest symbolic ' - for src in $sources; do - pfx=$(echo $src | sed -r 's/[^/]//g' | sed 's/\//..\//g') - if [ ! -e $src ]; then - echo "Creating symlink: $src" - ( cd "$(dirname $src)" && ln -s "$pfx$mm_repo/$src" ) + while read path type; do + [ "$path" ] || continue + pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') + symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) + target="$mm_repo/$path" + if [ ! -e "$target" ]; then + echo "Target path $target is missing! Cannot proceed" + exit 1 fi - done + fs="%-8s %-16s %s -> %s\n" + if [ $type == 'hard' ]; then + if [ -L $path ]; then + printf "$fs" "Deleting" "symbolic link:" $path $target + rm -rf $path + elif [ -e $path ]; then + if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then + printf "$fs" "Deleting" "stale hard link:" $path "?" + rm -rf $path + fi + fi + fi + if [ ! -e $path ]; then # link is either absent or a broken symlink + printf "$fs" "Creating" "$type link:" $path $target + ( cd "$(dirname $path)" && ln -f $symlink_arg $pfx$target ) + fi + done <<<$paths } set -e diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index cfe98b7..54d8edd 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -41,20 +41,20 @@ init_tests() { t_unit="- $unit_tests_py" d_misc="miscellaneous features" - t_misc="- $test_py helpscreens" + t_misc="- $cmdtest_py helpscreens" d_scripts="scripts not requiring a coin daemon" - t_scripts="- $test_py scripts" + t_scripts="- $cmdtest_py scripts" d_btc="Bitcoin with emulated RPC data" - t_btc="- $test_py main" + t_btc="- $cmdtest_py main" d_btc_rt="Bitcoin regtest" - t_btc_rt="- $test_py regtest" + t_btc_rt="- $cmdtest_py regtest" d_bch_rt="Bitcoin Cash Node (BCH) regtest" - t_bch_rt="- $test_py --coin=bch regtest" + t_bch_rt="- $cmdtest_py --coin=bch regtest" d_ltc_rt="Litecoin regtest" - t_ltc_rt="- $test_py --coin=ltc regtest" + t_ltc_rt="- $cmdtest_py --coin=ltc regtest" } From 3f76be1ab8e948a341688522573767f17fe5f9aa Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 13 Oct 2023 09:50:15 +0000 Subject: [PATCH 06/81] minor fixes and cleanups --- mmgen_node_tools/BlocksInfo.py | 93 +++++++++++++++++--------------- mmgen_node_tools/PollDisplay.py | 2 +- mmgen_node_tools/Ticker.py | 6 ++- mmgen_node_tools/main_addrbal.py | 2 + mmgen_node_tools/main_netrate.py | 4 +- mmgen_node_tools/main_ticker.py | 10 ++-- mmgen_node_tools/main_txfind.py | 4 +- setup.cfg | 1 + test/cmdtest_py_d/ct_regtest.py | 2 +- 9 files changed, 68 insertions(+), 56 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 25e1caf..40cc6e5 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -27,6 +27,52 @@ from time import strftime,gmtime from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int from mmgen.rpc import json_encoder +class RangeParser: + + debug = False + + def __init__(self,caller,arg): + self.caller = caller + self.arg = self.orig_arg = arg + + def parse(self,target): + ret = getattr(self,'parse_'+target)() + if self.debug: + msg(f'arg after parse({target}): {self.arg}') + return ret + + def finalize(self): + if self.arg: + die(1,f'{self.orig_arg!r}: invalid range specifier') + + def parse_from_tip(self): + m = re.match(r'-([0-9]+)(.*)',self.arg) + if m: + res,self.arg = (m[1],m[2]) + return self.caller.check_nblocks(int(res)) + + def parse_abs_range(self): + m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',self.arg) + if m: + if self.debug: + msg(f'abs_range parse: first={m[1]}, last={m[3]}') + self.arg = m[4] + return ( + self.caller.conv_blkspec(m[1]), + self.caller.conv_blkspec(m[3]) if m[3] else None + ) + return (None,None) + + def parse_add(self): + m = re.match(r'\+([0-9*]+)(.*)',self.arg) + if m: + res,self.arg = (m[1],m[2]) + if res.strip('*') != res: + die(1,f"'+{res}': malformed nBlocks specifier") + if len(res) > 30: + die(1,f"'+{res}': overly long nBlocks specifier") + return self.caller.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe + class BlocksInfo: total_bytes = 0 @@ -298,49 +344,8 @@ class BlocksInfo: def parse_rangespec(self,arg): - class RangeParser: - debug = False + p = RangeParser(self,arg) - def __init__(rp,arg): - rp.arg = rp.orig_arg = arg - - def parse(rp,target): - ret = getattr(rp,'parse_'+target)() - if rp.debug: msg(f'arg after parse({target}): {rp.arg}') - return ret - - def finalize(rp): - if rp.arg: - die(1,f'{rp.orig_arg!r}: invalid range specifier') - - def parse_from_tip(rp): - m = re.match(r'-([0-9]+)(.*)',rp.arg) - if m: - res,rp.arg = (m[1],m[2]) - return self.check_nblocks(int(res)) - - def parse_abs_range(rp): - m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',rp.arg) - if m: - if rp.debug: msg(f'abs_range parse: first={m[1]}, last={m[3]}') - rp.arg = m[4] - return ( - self.conv_blkspec(m[1]), - self.conv_blkspec(m[3]) if m[3] else None - ) - return (None,None) - - def parse_add(rp): - m = re.match(r'\+([0-9*]+)(.*)',rp.arg) - if m: - res,rp.arg = (m[1],m[2]) - if res.strip('*') != res: - die(1,f"'+{res}': malformed nBlocks specifier") - if len(res) > 30: - die(1,f"'+{res}': overly long nBlocks specifier") - return self.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe - - p = RangeParser(arg) from_tip = p.parse('from_tip') first,last = (self.tip-from_tip,None) if from_tip else p.parse('abs_range') add1 = p.parse('add') @@ -708,8 +713,8 @@ class BlocksInfo: class JSONBlocksInfo(BlocksInfo): - def __init__(self,cfg,cmd_args,opt,rpc): - super().__init__(cfg,cmd_args,opt,rpc) + def __init__(self,cfg,cmd_args,rpc): + super().__init__(cfg,cmd_args,rpc) if self.cfg.json_raw: self.output_block = self.output_block_raw self.fmt_stat_item = self.fmt_stat_item_raw diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 84a6023..5bbfecd 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -16,7 +16,7 @@ import sys,threading from mmgen.util import msg from mmgen.term import get_char -class PollDisplay(): +class PollDisplay: info = None input = None diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 97b5dc5..1e21192 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -449,7 +449,7 @@ def main(): '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n' ) -def make_cfg(): +def make_cfg(gcfg_arg): query_tuple = namedtuple('query',['asset','to_asset']) asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) @@ -578,7 +578,9 @@ def make_cfg(): 'proxy2', 'portfolio' ]) - global cfg_in,src_cls,cfg + global gcfg,cfg_in,src_cls,cfg + + gcfg = gcfg_arg src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } fi_pat = src_cls['fi'].asset_id_pat diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index a461e9e..34c6236 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -12,6 +12,8 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain """ +import sys + from mmgen.obj import CoinTxID,Int from mmgen.cfg import Config from mmgen.util import msg,Msg,die,suf,make_timestr,async_run diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index f1661c5..52b509d 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -49,7 +49,7 @@ async def main(): d = await c.call('getnettotals') return [float(e) for e in (d['totalbytesrecv'],d['totalbytessent'],d['timemillis'])] - rs = None + rs,ss,ts = (None,None,None) while True: r,s,t = await get_data() @@ -64,7 +64,7 @@ async def main(): if rs is not None: sys.stderr.write('{}{}{}'.format(ERASE_LINE,CUR_UP,ERASE_LINE)) - rs,ss,ts = r,s,t + rs,ss,ts = (r,s,t) try: async_run(main()) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index f329e03..6a9d3aa 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -216,14 +216,14 @@ import os from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config -import mmgen_node_tools.Ticker as tck +from . import Ticker -tck.gcfg = Config( opts_data=opts_data, do_post_init=True ) +gcfg = Config( opts_data=opts_data, do_post_init=True ) -tck.make_cfg() +Ticker.make_cfg(gcfg) from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls -tck.gcfg._post_init() +gcfg._post_init() -tck.main() +Ticker.main() diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 57b691a..4a54430 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -20,8 +20,10 @@ mmnode-txfind: Find a transaction in the blockchain or mempool """ +import sys + from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,is_hex_str +from mmgen.util import msg,Msg,die,is_hex_str,async_run opts_data = { 'text': { diff --git a/setup.cfg b/setup.cfg index aeebff5..7e91ea5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ include_package_data = True install_requires = mmgen>=14.0.dev6 + pyyaml yahooquery packages = diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index a03e3ff..b17b4c6 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -107,7 +107,7 @@ class CmdTestRegtest(CmdTestBase): CmdTestBase.__init__(self,trunner,cfgs,spawn) if trunner == None: return - if self.proto.testnet: + if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) From ada2cecadae5807678f27d34f8a871a49cfa9c65 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 13 Oct 2023 09:50:16 +0000 Subject: [PATCH 07/81] Check the MMGen Node Tools with the pylint static code analyzer: # Install pylint: $ python3 -m pip install pylint # Perform the check: $ test/test-release.sh lint The linting operation should complete without error if running Python v3.9 or greater. --- mmgen_node_tools/data/version | 2 +- pyproject.toml | 22 ++++++++++++++++++++++ setup.cfg | 2 +- test/test-release.d/cfg.sh | 16 ++++++++++++---- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4b653e3..5223fbd 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev2 +3.2.dev3 diff --git a/pyproject.toml b/pyproject.toml index 374b58c..81515d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,3 +4,25 @@ requires = [ "wheel" ] build-backend = "setuptools.build_meta" + +[tool.pylint.format] +indent-string = "\t" +indent-after-paren = 2 +max-line-length = 110 + +[tool.pylint.main] +py-version = "3.7" +recursive = true +jobs = 0 + +[tool.pylint."messages control"] +ignored-modules = [ + "mmgen.term", + "mmgen.color", +] +ignored-classes = [ + "mmgen_node_tools.Ticker.Ticker.base", + "mmgen_node_tools.Ticker.DataSource.base", + "mmgen_node_tools.PeerBlocks.Display", + "mmgen_node_tools.PollDisplay.PollDisplay", +] diff --git a/setup.cfg b/setup.cfg index 7e91ea5..55c315a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.7 include_package_data = True install_requires = - mmgen>=14.0.dev6 + mmgen>=14.0.dev9 pyyaml yahooquery diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 54d8edd..b371e72 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -18,7 +18,7 @@ # mmnode-ticker OK # mmnode-txfind - -all_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' +all_tests='unit lint misc scripts btc btc_rt bch_rt ltc_rt' groups_desc=" default - All tests minus the extra tests @@ -29,14 +29,22 @@ groups_desc=" " init_groups() { - dfl_tests=$all_tests - extra_tests='' + dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + extra_tests='lint' noalt_tests='unit misc scripts btc btc_rt' quick_tests='unit misc scripts btc btc_rt' - qskip_tests='bch_rt ltc_rt' + qskip_tests='lint bch_rt ltc_rt' } init_tests() { + + d_lint="code errors with static code analyzer" + t_lint=" + - $pylint --errors-only mmgen_node_tools + - $pylint --errors-only test + - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_py_d + " + d_unit="low-level subsystems" t_unit="- $unit_tests_py" From e94f036c73abe57b41e1ffbf3f715198249fddda Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:22 +0000 Subject: [PATCH 08/81] mmnode-ticker: cleanups --- mmgen_node_tools/Ticker.py | 80 ++++++++++++++++++++------------- mmgen_node_tools/main_ticker.py | 14 +++--- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 1e21192..fc0b8da 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -32,26 +32,34 @@ dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) - -def fetch_delay(fetched_data=[]): - if not gcfg.testing: - if fetched_data: - delay = 1 + random.randrange(1,5000) / 1000 - msg_r(f'Waiting {delay:.3f} seconds...') - time.sleep(delay) - msg('') - else: - fetched_data.append(None) +last_api_host = None class DataSource: - sources = { - 'cc': 'coinpaprika', - 'fi': 'yahooquery' - } + source_groups = [ + { + 'cc': 'coinpaprika' + }, { + 'fi': 'yahoospot', + } + ] + + @classmethod + def get_sources(cls,randomize=False): + g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups + return {k:v for a in g for k,v in a.items()} class base: + def fetch_delay(self): + global last_api_host + if not gcfg.testing and last_api_host and last_api_host != self.api_host: + delay = 1 + random.randrange(1,5000) / 1000 + msg_r(f'Waiting {delay:.3f} seconds...') + time.sleep(delay) + msg('') + last_api_host = self.api_host + def get_data_from_network(self): curl_cmd = list_gen( @@ -90,11 +98,11 @@ class DataSource: else: data_type = self.net_data_type elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) - if elapsed >= self.timeout: + if elapsed >= self.timeout or gcfg.testing: if gcfg.testing: msg('') - fetch_delay() - msg_r(f'Fetching data from {self.api_host}...') + self.fetch_delay() + msg_r(f'Fetching {self.data_desc} from {self.api_host}...') if self.has_verbose: gcfg._util.vmsg('') data_in = self.get_data_from_network() @@ -145,6 +153,7 @@ class DataSource: class coinpaprika(base): desc = 'CoinPaprika' + data_desc = 'cryptocurrency data' api_host = 'api.coinpaprika.com' ratelimit = 240 btc_ratelimit = 10 @@ -201,14 +210,16 @@ class DataSource: id = (s.lower() if label else None), source = 'cc' ) - class yahooquery(base): + class yahoospot(base): desc = 'Yahoo Finance' + data_desc = 'spot financial data' api_host = 'finance.yahoo.com' ratelimit = 30 net_data_type = 'python' has_verbose = False asset_id_pat = r'^\^.*|.*=[xf]$' + json_fn_basename = 'ticker-finance.json' @staticmethod def get_id(sym,data): @@ -233,29 +244,37 @@ class DataSource: @property def json_fn(self): - return os.path.join( cfg.cachedir, 'ticker-finance.json' ) + return os.path.join( cfg.cachedir, self.json_fn_basename ) @property def timeout(self): return 5 if gcfg.test_suite else self.ratelimit + @property + def symbols(self): + return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + def get_data_from_network(self): - arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] - - kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } } + kwargs = { + 'formatted': True, + 'proxies': { 'https': cfg.proxy2 }, + } if gcfg.test_suite: kwargs.update({ 'timeout': 1, 'retry': 0 }) if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( - arg, + self.symbols, fmt_dict(kwargs,fmt='kwargs') )) return from yahooquery import Ticker - return Ticker(arg,**kwargs).price + return self.process_network_data( Ticker(self.symbols,**kwargs) ) + + def process_network_data(self,ticker): + return ticker.price @staticmethod def parse_asset_id(s,require_label): @@ -422,15 +441,13 @@ def main(): if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: - if not gcfg.download in DataSource.sources: + if not gcfg.download in DataSource.get_sources(): die(1,f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: - src_ids = DataSource.sources + src_ids = DataSource.get_sources(randomize=True) - ids = random.sample( list(src_ids), k=len(src_ids) ) # shuffle the ids - - src_data = { k: src_cls[k]().get_data() for k in ids } + src_data = { k: src_cls[k]().get_data() for k in src_ids } if gcfg.testing: return @@ -582,7 +599,7 @@ def make_cfg(gcfg_arg): gcfg = gcfg_arg - src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } + src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() } fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args @@ -673,8 +690,7 @@ class Ticker: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) # ticker API - # t.replace('T',' ').replace('Z','') # tickers API + fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) d = self.data max_w = 0 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 6a9d3aa..13175b8 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -14,10 +14,10 @@ mmnode-ticker: Display price information for cryptocurrency and other assets opts_data = { 'sets': [ - ('wide', True, 'percent_change', True), - ('wide', True, 'name_labels', True), - ('wide', True, 'thousands_comma', True), - ('wide', True, 'update_time', True), + ('wide', True, 'percent_change', True), + ('wide', True, 'name_labels', True), + ('wide', True, 'thousands_comma', True), + ('wide', True, 'update_time', True), ], 'text': { 'desc': 'Display prices for cryptocurrency and other assets', @@ -35,8 +35,8 @@ opts_data = { live data from server -D, --cachedir=D Read and write cached JSON data to directory ‘D’ instead of ‘~/{dfl_cachedir}’ --d, --download=D Retrieve data ‘D’ from source, save to file and exit - (valid options: {ds}) +-d, --download=D Retrieve and cache asset data ‘D’ from network (valid + options: {ds}) -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -200,7 +200,7 @@ To add a portfolio, edit the file 'code': { 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), - ds = fmt_dict(DataSource.sources,fmt='equal'), + ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), From 94c61d680702cc1139d75c58f3105cad1b05446b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 09/81] mmnode-ticker: convert amounts to type Decimal on input --- mmgen_node_tools/Ticker.py | 48 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index fc0b8da..b2fc86a 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -232,8 +232,8 @@ class DataSource: 'id': sym, 'name': data['shortName'], 'symbol': sym.upper(), - 'price_usd': str(price_usd), - 'price_btc': str(price_usd / btcusd), + 'price_usd': price_usd, + 'price_btc': price_usd / btcusd, 'percent_change_7d': None, 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, 'last_updated': data['regularMarketTime'], @@ -378,6 +378,13 @@ def gen_data(data): if d[k] in wants[k]: if d[k] in found[k]: die(1,dup_sym_errmsg(d[k])) + if not d.get('_converted'): + d['price_usd'] = Decimal(d['price_usd']) + d['price_btc'] = Decimal(d['price_btc']) + d['percent_change_7d'] = Decimal(d['percent_change_7d']) + d['percent_change_24h'] = Decimal(d['percent_change_24h']) + d['last_updated'] = int(d['last_updated']) + d['_converted'] = True yield (d['id'],d) found[k].add(d[k]) wants[k].remove(d[k]) @@ -394,13 +401,13 @@ def gen_data(data): User-supplied rate overrides rate from source data. """ _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() - ra_rate = Decimal(rate_assets[asset.rate_asset.symbol]['price_usd']) if asset.rate_asset else 1 + ra_rate = rate_assets[asset.rate_asset.symbol]['price_usd'] if asset.rate_asset else 1 yield ( _id, { 'symbol': asset.symbol, 'id': _id, 'name': ' '.join(_id.split('-')[1:]), - 'price_usd': str(Decimal(ra_rate/asset.rate)), - 'price_btc': str(Decimal(ra_rate/asset.rate/btcusd)), + 'price_usd': ra_rate / asset.rate, + 'price_btc': ra_rate / asset.rate / btcusd, 'last_updated': None, }) @@ -408,8 +415,8 @@ def gen_data(data): 'symbol': 'USD', 'id': 'usd-us-dollar', 'name': 'US Dollar', - 'price_usd': '1.0', - 'price_btc': str(Decimal(1/btcusd)), + 'price_usd': Decimal(1), + 'price_btc': Decimal(1) / btcusd, 'last_updated': None, }) @@ -678,7 +685,7 @@ class Ticker: )) + 1 self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows] - self.col_usd_prices = {k:Decimal(self.data[k]['price_usd']) for k in self.col_ids} + self.col_usd_prices = {k:self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id:self.get_row_prices(row.id) for row in self.rows if isinstance(row,tuple) and row.id in data} @@ -717,7 +724,7 @@ class Ticker: self.upd_w = max_w def init_prec(self): - exp = [(a.id,Decimal.adjusted(self.prices[a.id]['usd-us-dollar'])) for a in self.usr_col_assets] + exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted() ) for a in self.usr_col_assets] self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp } self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp } @@ -737,13 +744,13 @@ class Ticker: for asset in self.usr_col_assets: if asset.symbol != 'USD': - usdprice = Decimal(self.data[asset.id]['price_usd']) + usdprice = self.data[asset.id]['price_usd'] yield '{} ({}) = {:{}.{}f} USD'.format( asset.symbol, self.create_label(asset.id), usdprice, self.comma, - max(2,int(-usdprice.adjusted())+4) ) + max(2, 4-usdprice.adjusted()) ) if hasattr(self,'subhdr'): yield self.subhdr @@ -818,17 +825,14 @@ class Ticker: if id in self.data: d = self.data[id] return { k: ( - Decimal(d['price_btc']) if k == 'btc-bitcoin' else - Decimal(d['price_usd']) / self.col_usd_prices[k] + d['price_btc'] if k == 'btc-bitcoin' else + d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids } def fmt_row(self,d,amt=None,amt_fmt=None): - def fmt_pct(d): - if d in ('',None): - return gray(' --') - n = Decimal(d) - return (red,green)[n>=0](f'{n:+7.2f}') + def fmt_pct(n): + return gray(' --') if n == None else (red,green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] @@ -927,8 +931,8 @@ class Ticker: if self.offer: real_price = ( self.asset.amount - * Decimal(data[self.asset.id]['price_usd']) - / Decimal(data[self.to_asset.id]['price_usd']) + * data[self.asset.id]['price_usd'] + / data[self.to_asset.id]['price_usd'] ) if self.adjust != 1: die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier') @@ -943,7 +947,7 @@ class Ticker: self.usr_col_assets = [self.asset] + ([self.to_asset] if self.to_asset else []) for a in self.usr_col_assets: - self.prices[a.id]['usd-us-dollar'] = Decimal(data[a.id]['price_usd']) + self.prices[a.id]['usd-us-dollar'] = data[a.id]['price_usd'] self.format_last_update_col(cross_assets=self.usr_col_assets) @@ -953,7 +957,7 @@ class Ticker: def get_row_prices(self,id): if id in self.data: d = self.data[id] - return { k: self.col_usd_prices[self.asset.id] / Decimal(d['price_usd']) for k in self.col_ids } + return { k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids } def init_fs(self): self.max_wid = max( From 77d8ffb7b9accfa086a1b8e89f494b393c2940de Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 10/81] mmnode-ticker: use coinpaprika `tickers` API call --- mmgen_node_tools/Ticker.py | 28 +++++++++++++++++----------- mmgen_node_tools/main_ticker.py | 20 +++++++++++++------- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 12 ++++++------ test/ref/ticker/ticker-btc.json | 2 +- test/ref/ticker/ticker.json | 2 +- 6 files changed, 39 insertions(+), 27 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index b2fc86a..40bede8 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -12,14 +12,14 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets """ -# We use deprecated coinpaprika ‘ticker’ API for now because it returns ~45% less data. +# v3.2.dev4: switch to new coinpaprika ‘tickers’ API call (supports ‘limit’ parameter, more historical data) # Old ‘ticker’ API (/v1/ticker): data['BTC']['price_usd'] # New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price'] # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,re,time,json,yaml,random +import sys,os,re,time,datetime,json,yaml,random from subprocess import run,PIPE,CalledProcessError from decimal import Decimal from collections import namedtuple @@ -159,6 +159,10 @@ class DataSource: btc_ratelimit = 10 net_data_type = 'json' has_verbose = True + dfl_asset_limit = 2000 + + def __init__(self): + self.asset_limit = int(gcfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self,elapsed): return ( @@ -168,7 +172,10 @@ class DataSource: @property def api_url(self): - return f'https://{self.api_host}/v1/ticker' + ('/btc-bitcoin' if cfg.btc_only else '') + return ( + f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else + f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else + f'https://{self.api_host}/v1/tickers' ) @property def json_fn(self): @@ -352,7 +359,7 @@ def gen_data(data): for d in data['cc']: if d['id'] == 'btc-bitcoin': - btcusd = Decimal(d['price_usd']) + btcusd = Decimal(str(d['quotes']['USD']['price'])) break get_id = src_cls['fi'].get_id @@ -378,13 +385,12 @@ def gen_data(data): if d[k] in wants[k]: if d[k] in found[k]: die(1,dup_sym_errmsg(d[k])) - if not d.get('_converted'): - d['price_usd'] = Decimal(d['price_usd']) - d['price_btc'] = Decimal(d['price_btc']) - d['percent_change_7d'] = Decimal(d['percent_change_7d']) - d['percent_change_24h'] = Decimal(d['percent_change_24h']) - d['last_updated'] = int(d['last_updated']) - d['_converted'] = True + if not 'price_usd' in d: + d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) + d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd + d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] + d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp()) yield (d['id'],d) found[k].add(d[k]) wants[k].remove(d[k]) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 13175b8..91bc293 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -25,6 +25,9 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) +-a, --asset-limit=N Retrieve data for top ‘N’ cryptocurrencies by market + cap (default: {al}). To retrieve all available data, + specify a value of zero. -A, --adjust=P Adjust prices by percentage ‘P’. In ‘trading’ mode, spot and adjusted prices are shown in separate columns. -b, --btc Fetch and display data for Bitcoin only @@ -137,13 +140,14 @@ Financial data is obtained from {fi.desc}, which currently allows Tor. RATE LIMITING NOTE -To protect user privacy, all filtering and processing of cryptocurrency data -is performed client side so that the remote server does not know which assets -are being examined. This means that data for ALL available crypto assets -(currently over 8000) is fetched with each invocation of the script. A rate -limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the -remote server. When the --btc option is in effect, this limit is reduced to -{cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data. +To protect user privacy, filtering and processing of cryptocurrency data is +performed client side so that the remote server does not know which assets +are being examined. This is done by fetching data for the top {al} crypto +assets by market cap (configurable via the --asset-limit option) with each +invocation of the script. A rate limit of {cc.ratelimit} seconds between calls is thus +imposed to prevent abuse of the remote server. When the --btc option is in +effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the rate limit +entirely, use --cached-data. Note that financial data obtained from {fi.api_host} is filtered in the request, which has privacy implications. The rate limit for financial data @@ -201,11 +205,13 @@ To add a portfolio, edit the file 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), + al = DataSource.coinpaprika.dfl_asset_limit, ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), fi = src_cls['fi'](), ) diff --git a/setup.cfg b/setup.cfg index 55c315a..0c1e362 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ classifiers = Operating System :: Microsoft :: Windows [options] -python_requires = >=3.7 +python_requires = >=3.8 include_package_data = True install_requires = diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 966fb91..3a9c47c 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -132,7 +132,7 @@ class CmdTestScripts(CmdTestBase): [], [ 'USD BTC', - 'BTC 23250.77 1.00000000 ETH 1659.66 0.07146397' + 'BTC 23250.77 1.00000000 ETH 1659.66 0.07138094' ]) @@ -144,8 +144,8 @@ class CmdTestScripts(CmdTestBase): r'INR \(INDIAN RUPEE\) = 0.012579 USD', 'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED', 'BITCOIN', - r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07146397 \+21.42 \+1.82', - r'MONERO 158.97 149.3870 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59', + r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+21.42 \+1.82', + r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+7.28 \+1.21 2022-08-02 18:25:59', r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --', ]) @@ -156,8 +156,8 @@ class CmdTestScripts(CmdTestBase): [ 'Adjusting prices by -0.77%', 'USD BTC CHG_7d CHG_24h UPDATED', - r'LITECOIN 58.56 0.00252162 \+12.79 \+0.40 2022-08-02 18:25:59', - r'MONERO 157.76 0.00679284 \+7.28 \+1.21' + r'LITECOIN 58.56 0.00251869 \+12.79 \+0.40 2022-08-02 18:25:59', + r'MONERO 157.76 0.00678495 \+7.28 \+1.21' ]) os.unlink(os.path.join(self.nt_datadir,'ticker-cfg.yaml')) return t @@ -174,7 +174,7 @@ class CmdTestScripts(CmdTestBase): ['--wide','--portfolio'], [ 'USD BTC CHG_7d CHG_24h UPDATED', - r'ETHEREUM 1,659.66 0.07146397 \+21.42 \+1.82 2022-08-02 18:25:59', + r'ETHEREUM 1,659.66 0.07138094 \+21.42 \+1.82 2022-08-02 18:25:59', 'CARDANO','ALGORAND', 'PORTFOLIO','BITCOIN','ETHEREUM','MONERO','CARDANO','ALGORAND','TOTAL' ]) diff --git a/test/ref/ticker/ticker-btc.json b/test/ref/ticker/ticker-btc.json index 75c8909..d0549b1 100644 --- a/test/ref/ticker/ticker-btc.json +++ b/test/ref/ticker/ticker-btc.json @@ -1 +1 @@ -{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23368.859557988893","price_btc":"1","volume_24h_usd":"24116251608.791744","market_cap_usd":"446560795287","circulating_supply":"19109225","total_supply":"19109231","max_supply":"21000000","percent_change_1h":"-0.23","percent_change_24h":"-1.87","percent_change_7d":"6.05","last_updated":"1659346445"} \ No newline at end of file +{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","circulating_supply":"19109225","total_supply":"19109231","max_supply":"21000000","last_updated":"2022-08-01T09:34:05Z","quotes":{"USD":{"price":23368.859557988893,"percent_change_1h":-0.23,"percent_change_6h":-1.4960000000000002,"percent_change_24h":-1.87,"percent_change_7d":6.05,"percent_change_30d":8.469999999999999,"percent_change_1y":10.285,"volume_24h":24116251608.791744,"market_cap":446560795287}}} diff --git a/test/ref/ticker/ticker.json b/test/ref/ticker/ticker.json index 880a666..4535b12 100644 --- a/test/ref/ticker/ticker.json +++ b/test/ref/ticker/ticker.json @@ -1 +1 @@ -[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","price_usd":"23250.774053363122","price_btc":"1","volume_24h_usd":"28307579008.4866","market_cap_usd":"444336521633","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","percent_change_1h":"-0.27","percent_change_24h":"0.89","percent_change_7d":"11.15","last_updated":"1659464759"},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","price_usd":"1659.6621665887371","price_btc":"0.07146396683507168","volume_24h_usd":"18216561308.363518","market_cap_usd":"202151289827","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","percent_change_1h":"-0.03","percent_change_24h":"1.82","percent_change_7d":"21.42","last_updated":"1659464759"},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","price_usd":"59.013589192027936","price_btc":"0.002541086532993605","volume_24h_usd":"510336800.72056556","market_cap_usd":"4181502638","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","percent_change_1h":"-0.43","percent_change_24h":"0.4","percent_change_7d":"12.79","last_updated":"1659464759"},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","price_usd":"158.97302629813817","price_btc":"0.006845274482813666","volume_24h_usd":"78159392.30003875","market_cap_usd":"2886277702","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","percent_change_1h":"0.21","percent_change_24h":"1.21","percent_change_7d":"7.28","last_updated":"1659464759"},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","price_usd":"0.5069722536476557","price_btc":"0.00002182989348696495","volume_24h_usd":"507973898.04662097","market_cap_usd":"17111613980","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","percent_change_1h":"0.05","percent_change_24h":"-0.11","percent_change_7d":"11.68","last_updated":"1659464759"},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","price_usd":"0.33196893931152616","price_btc":"0.00001429436529121747","volume_24h_usd":"62207779.35759487","market_cap_usd":"2306909784","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","percent_change_1h":"-0.06","percent_change_24h":"-0.82","percent_change_7d":"9.69","last_updated":"1659464759"}] +[{"id":"btc-bitcoin","name":"Bitcoin","symbol":"BTC","rank":"1","circulating_supply":"19110612","total_supply":"19110619","max_supply":"21000000","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":23250.774053363122,"percent_change_1h":-0.27,"percent_change_6h":0.7120000000000001,"percent_change_24h":0.89,"percent_change_7d":11.15,"percent_change_30d":15.61,"percent_change_1y":18.955000000000002,"volume_24h":28307579008.4866,"market_cap":444336521633}}},{"id":"eth-ethereum","name":"Ethereum","symbol":"ETH","rank":"2","circulating_supply":"121802674","total_supply":"121802721","max_supply":"","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":1659.6621665887371,"percent_change_1h":-0.03,"percent_change_6h":1.4560000000000002,"percent_change_24h":1.82,"percent_change_7d":21.42,"percent_change_30d":29.988,"percent_change_1y":36.414,"volume_24h":18216561308.363518,"market_cap":202151289827}}},{"id":"ltc-litecoin","name":"Litecoin","symbol":"LTC","rank":"23","circulating_supply":"70856606","total_supply":"70856631","max_supply":"84000000","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":59.013589192027936,"percent_change_1h":-0.43,"percent_change_6h":0.32000000000000006,"percent_change_24h":0.4,"percent_change_7d":12.79,"percent_change_30d":17.906,"percent_change_1y":21.743,"volume_24h":510336800.72056556,"market_cap":4181502638}}},{"id":"xmr-monero","name":"Monero","symbol":"XMR","rank":"30","circulating_supply":"18155770","total_supply":"18155768","max_supply":"","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":158.97302629813817,"percent_change_1h":0.21,"percent_change_6h":0.968,"percent_change_24h":1.21,"percent_change_7d":7.28,"percent_change_30d":10.192,"percent_change_1y":12.376,"volume_24h":78159392.30003875,"market_cap":2886277702}}},{"id":"ada-cardano","name":"Cardano","symbol":"ADA","rank":"8","circulating_supply":"33752565071","total_supply":"34277702082","max_supply":"45000000000","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":0.5069722536476557,"percent_change_1h":0.05,"percent_change_6h":-0.08800000000000001,"percent_change_24h":-0.11,"percent_change_7d":11.68,"percent_change_30d":16.352,"percent_change_1y":19.855999999999998,"volume_24h":507973898.04662097,"market_cap":17111613980}}},{"id":"algo-algorand","name":"Algorand","symbol":"ALGO","rank":"33","circulating_supply":"6949173585","total_supply":"7350412337","max_supply":"","last_updated":"2022-08-02T18:25:59Z","quotes":{"USD":{"price":0.33196893931152616,"percent_change_1h":-0.06,"percent_change_6h":-0.656,"percent_change_24h":-0.82,"percent_change_7d":9.69,"percent_change_30d":13.565999999999999,"percent_change_1y":16.473,"volume_24h":62207779.35759487,"market_cap":2306909784}}}] From 545bc044c6e89d48e814cb64e8649ffbba637d85 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 11/81] mmnode-ticker: add year and month percentage change columns --- mmgen_node_tools/Ticker.py | 42 +++++++++++++++++++++++++++------ mmgen_node_tools/main_ticker.py | 21 ++++++++++++----- test/cmdtest_py_d/ct_misc.py | 11 +++++---- 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 40bede8..4011c90 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -34,6 +34,13 @@ portfolio_fn = 'ticker-portfolio.yaml' asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) last_api_host = None +percent_cols = { + 'd': 'day', + 'w': 'week', + 'm': 'month', + 'y': 'year', +} + class DataSource: source_groups = [ @@ -390,6 +397,8 @@ def gen_data(data): d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] + d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp()) yield (d['id'],d) found[k].add(d[k]) @@ -497,6 +506,15 @@ def make_cfg(gcfg_arg): yield parse_asset_id(e,require_label=True) return tuple(gen()) + def parse_percent_cols(arg): + if arg is None: + return [] + res = arg.lower().split(',') + for s in res: + if s not in percent_cols: + die(1,f'{arg!r}: invalid --percent-cols parameter (valid letters: {fmt_list(percent_cols)})') + return res + def parse_usr_asset_arg(key,use_cf_file=False): """ asset_id[:rate[:rate_asset]] @@ -606,7 +624,8 @@ def make_cfg(gcfg_arg): 'cachedir', 'proxy', 'proxy2', - 'portfolio' ]) + 'portfolio', + 'percent_cols' ]) global gcfg,cfg_in,src_cls,cfg @@ -645,7 +664,8 @@ def make_cfg(gcfg_arg): cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None + portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None, + percent_cols = parse_percent_cols(gcfg.percent_cols) ) def get_cfg_in(): @@ -802,7 +822,7 @@ class Ticker: yield '-' * self.hl_wid if not cfg.btc_only: yield self.fs_num.format( - lbl = 'TOTAL', pc1='', pc2='', upd='', amt='', + lbl = 'TOTAL', pc3='', pc4='', pc1='', pc2='', upd='', amt='', **{ k.replace('-','_'): v for k,v in self.prices['total'].items() } ) @@ -851,6 +871,8 @@ class Ticker: lbl = (self.create_label(d['id']) if gcfg.name_labels else d['symbol']), pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), + pc3 = fmt_pct(d.get('percent_change_1y')), + pc4 = fmt_pct(d.get('percent_change_30d')), upd = d.get('last_updated_fmt'), amt = amt_fmt, **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() } @@ -873,8 +895,10 @@ class Ticker: col_fs_data = { 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid), - 'pct7d': fd(' {pc1:7}', ' {pc1:7}', 8), - 'pct24h': fd(' {pc2:7}', ' {pc2:7}', 8), + 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), + 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), + 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), + 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2), 'amt': fd(' {amt}', ' {amt}', 21), } @@ -891,8 +915,10 @@ class Ticker: [asset.id for asset in self.usr_col_assets] + [a for a,b in ( ( 'btc-bitcoin', not cfg.btc_only ), - ( 'pct7d', gcfg.percent_change ), - ( 'pct24h', gcfg.percent_change ), + ( 'pct1y', 'y' in cfg.percent_cols ), + ( 'pct1m', 'm' in cfg.percent_cols ), + ( 'pct1w', 'w' in cfg.percent_cols ), + ( 'pct1d', 'd' in cfg.percent_cols ), ( 'update_time', gcfg.update_time ), ) if b] ) @@ -915,6 +941,8 @@ class Ticker: lbl = '', pc1 = ' CHG_7d', pc2 = 'CHG_24h', + pc3 = 'CHG_1y', + pc4 = 'CHG_30d', upd = 'UPDATED', amt = ' AMOUNT', usd_us_dollar = 'USD', diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 91bc293..eafa34a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -14,7 +14,11 @@ mmnode-ticker: Display price information for cryptocurrency and other assets opts_data = { 'sets': [ - ('wide', True, 'percent_change', True), + ('widest', True, 'percent_cols', 'd,w,m,y'), + ('widest', True, 'name_labels', True), + ('widest', True, 'thousands_comma', True), + ('widest', True, 'update_time', True), + ('wide', True, 'percent_cols', 'd,w'), ('wide', True, 'name_labels', True), ('wide', True, 'thousands_comma', True), ('wide', True, 'update_time', True), @@ -45,7 +49,9 @@ opts_data = { -F, --portfolio Display portfolio data -l, --list-ids List IDs of all available assets -n, --name-labels Label rows with asset names rather than symbols --p, --percent-change Add percentage change columns +-p, --percent-cols=C Add daily, weekly, monthly, or yearly percentage change + columns ‘C’ (specify with comma-separated letters + {pc}) -P, --pager Pipe the output to a pager -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply @@ -54,7 +60,8 @@ opts_data = { -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column -v, --verbose Be more verbose --w, --wide Display all optional columns (equivalent to -punT) +-w, --wide Display most optional columns (same as -unT -p d,w) +-W, --widest Display all optional columns (same as -unT -p d,w,m,y) -x, --proxy=P Connect via proxy ‘P’. Set to the empty string to completely disable or ‘none’ to allow override from environment. Consult the curl manpage for --proxy usage. @@ -169,9 +176,10 @@ $ mmnode-ticker -w -c eurusd=x,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118 # Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate: $ mmnode-ticker -wE -c eurusd=x,bgn-bulgarian-lev:0.5113r:eurusd=x -# Wide display, use cached data from previous network query, show portfolio -# (see above), pipe output to pager, add DOGE row: -$ mmnode-ticker -wCFP -r doge +# Widest display with all percentage change columns, use cached data from +# previous network query, show portfolio (see above), pipe output to pager, +# add DOGE row: +$ mmnode-ticker -WCFP -r doge # Display 17.234 XMR priced in all configured assets (‘trading’ mode): $ mmnode-ticker xmr:17.234 @@ -206,6 +214,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, + pc = fmt_list(Ticker.percent_cols,fmt='bare'), ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 3a9c47c..8046e51 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -138,15 +138,16 @@ class CmdTestScripts(CmdTestBase): def ticker4(self): return self.ticker( - ['--wide','--add-columns=eurusd=x,inr-indian-rupee:79.5'], + ['--widest','--add-columns=eurusd=x,inr-indian-rupee:79.5'], [ r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' + r'INR \(INDIAN RUPEE\) = 0.012579 USD', - 'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED', + 'USD EURUSD=X INR BTC CHG_1y CHG_30d CHG_7d CHG_24h UPDATED', 'BITCOIN', - r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+21.42 \+1.82', - r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+7.28 \+1.21 2022-08-02 18:25:59', - r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --', + r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+36.41 \+29.99 \+21.42 \+1.82', + r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+12.38 \+10.19 \+7.28 \+1.21 2022-08-02 18:25:59', + r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -- -- -- -0.23', + r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- -- -- --', ]) def ticker5(self): From af7c14fe369fc37d4d465e9a4bcb9f35b1952173 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH 12/81] mmnode-ticker: get historical financial data from Yahoo Finance --- mmgen_node_tools/Ticker.py | 43 ++++++++++++++++++++- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 3 +- test/ref/ticker/ticker-finance-history.json | 1 + 5 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 test/ref/ticker/ticker-finance-history.json diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 4011c90..6c15906 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -48,6 +48,7 @@ class DataSource: 'cc': 'coinpaprika' }, { 'fi': 'yahoospot', + 'hi': 'yahoohist', } ] @@ -248,7 +249,9 @@ class DataSource: 'symbol': sym.upper(), 'price_usd': price_usd, 'price_btc': price_usd / btcusd, - 'percent_change_7d': None, + 'percent_change_1y': data['pct_chg_1y'], + 'percent_change_30d': data['pct_chg_4wks'], + 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, 'last_updated': data['regularMarketTime'], } @@ -272,6 +275,7 @@ class DataSource: kwargs = { 'formatted': True, + 'asynchronous': True, 'proxies': { 'https': cfg.proxy2 }, } @@ -297,6 +301,33 @@ class DataSource: id = s.lower(), source = 'fi' ) + class yahoohist(yahoospot): + + json_fn_basename = 'ticker-finance-history.json' + data_desc = 'historical financial data' + net_data_type = 'json' + period = '1y' + interval = '1wk' + + def process_network_data(self,ticker): + return ticker.history( + period = self.period, + interval = self.interval).to_json(orient='index') + + def postprocess_data(self,data): + def gen(): + keys = set() + for key,val in data.items(): + if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): + date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) + if (sym := m[1]) in keys: + d[date] = val + else: + keys.add(sym) + d = {date:val} + yield (sym,d) + return dict(gen()) + def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield '' @@ -378,6 +409,16 @@ def gen_data(data): if id in wants['id']: if id in found['id']: die(1,dup_sym_errmsg(id)) + if m := data['hi'].get(k): + spot = v['regularMarketPrice']['raw'] + hist = tuple(m.values()) + v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( + (spot / hist[-2]['close'] - 1) * 100, + (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month + (spot / hist[0]['close'] - 1) * 100, + ) + else: + v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None yield ( id, conv_func(id,v,btcusd) ) found['id'].add(id) wants['id'].remove(id) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 5223fbd..ce7b703 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev3 +3.2.dev4 diff --git a/setup.cfg b/setup.cfg index 0c1e362..339e7f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen>=14.0.dev9 + mmgen>=14.0.dev11 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 8046e51..3dbac50 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -102,6 +102,7 @@ class CmdTestScripts(CmdTestBase): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) + shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' @@ -146,7 +147,7 @@ class CmdTestScripts(CmdTestBase): 'BITCOIN', r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+36.41 \+29.99 \+21.42 \+1.82', r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+12.38 \+10.19 \+7.28 \+1.21 2022-08-02 18:25:59', - r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -- -- -- -0.23', + r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -1.71 \+12.93 \+9.05 -0.23', r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- -- -- --', ]) diff --git a/test/ref/ticker/ticker-finance-history.json b/test/ref/ticker/ticker-finance-history.json new file mode 100644 index 0000000..e8b0bd3 --- /dev/null +++ b/test/ref/ticker/ticker-finance-history.json @@ -0,0 +1 @@ +{"('BZ=F', datetime.date(2021, 8, 2))":{"open":75.1800003052,"high":75.3300018311,"low":69.75,"close":70.6999969482,"volume":192386,"adjclose":70.6999969482},"('BZ=F', datetime.date(2022, 6, 27))":{"open":112.4599990845,"high":120.3799972534,"low":108.0100021362,"close":111.6299972534,"volume":64094,"adjclose":111.6299972534},"('BZ=F', datetime.date(2022, 7, 4))":{"open":111.6100006104,"high":114.6900024414,"low":98.4700012207,"close":107.0199966431,"volume":115623,"adjclose":107.0199966431},"('BZ=F', datetime.date(2022, 7, 11))":{"open":106.7600021362,"high":107.6600036621,"low":94.5,"close":101.1600036621,"volume":109537,"adjclose":101.1600036621},"('BZ=F', datetime.date(2022, 7, 18))":{"open":100.9199981689,"high":107.5999984741,"low":99.4800033569,"close":103.1999969482,"volume":110290,"adjclose":103.1999969482},"('BZ=F', datetime.date(2022, 7, 25))":{"open":103.4400024414,"high":110.4300003052,"low":101.6699981689,"close":110.0100021362,"volume":77572,"adjclose":110.0100021362},"('BZ=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":102.4400024414,"high":111.4300003052,"low":101.6699981689,"close":111.0100021362,"volume":77572,"adjclose":111.0100021362},"('CHFUSD=X', datetime.date(2021, 7, 26))":{"open":1.0939244032,"high":1.1064150333,"low":1.0908932686,"close":1.1047636271,"volume":0,"adjclose":1.1047636271},"('CHFUSD=X', datetime.date(2022, 7, 4))":{"open":1.043405652,"high":1.0446264744,"low":1.020783186,"close":1.0241703987,"volume":0,"adjclose":1.0241703987},"('CHFUSD=X', datetime.date(2022, 7, 11))":{"open":1.022777319,"high":1.0247056484,"low":1.0118077993,"close":1.0238809586,"volume":0,"adjclose":1.0238809586},"('CHFUSD=X', datetime.date(2022, 7, 18))":{"open":1.0250102282,"high":1.0417101383,"low":1.0216904879,"close":1.0396090746,"volume":0,"adjclose":1.0396090746},"('CHFUSD=X', datetime.date(2022, 7, 25))":{"open":1.0384216309,"high":1.0522992611,"low":1.0345113277,"close":1.0506275892,"volume":0,"adjclose":1.0506275892},"('CHFUSD=X', datetime.date(2022, 8, 1))":{"open":1.0492409468,"high":1.0547411442,"low":1.0495601892,"close":1.0492409468,"volume":0,"adjclose":1.0492409468},"('CHFUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.0492409468,"high":1.0547411442,"low":1.0495601892,"close":1.0492409468,"volume":0,"adjclose":1.0492409468},"('EURUSD=X', datetime.date(2021, 7, 26))":{"open":1.1822398901,"high":1.1911001205,"low":1.1785781384,"close":1.1867604256,"volume":0,"adjclose":1.1867604256},"('EURUSD=X', datetime.date(2022, 7, 4))":{"open":1.043394804,"high":1.0463534594,"low":1.0079730749,"close":1.0187449455,"volume":0,"adjclose":1.0187449455},"('EURUSD=X', datetime.date(2022, 7, 11))":{"open":1.0166114569,"high":1.0167768002,"low":0.9953616261,"close":1.008867979,"volume":0,"adjclose":1.008867979},"('EURUSD=X', datetime.date(2022, 7, 18))":{"open":1.0096318722,"high":1.0275379419,"low":1.0082373619,"close":1.0215548277,"volume":0,"adjclose":1.0215548277},"('EURUSD=X', datetime.date(2022, 7, 25))":{"open":1.0200231075,"high":1.0256409645,"low":1.0107442141,"close":1.0227040052,"volume":0,"adjclose":1.0227040052},"('EURUSD=X', datetime.date(2022, 8, 1))":{"open":1.02082479,"high":1.0277491808,"low":1.020960331,"close":1.02082479,"volume":0,"adjclose":1.02082479},"('EURUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.02082479,"high":1.0277491808,"low":1.020960331,"close":1.02082479,"volume":0,"adjclose":1.02082479},"('GBPUSD=X', datetime.date(2021, 7, 26))":{"open":1.3886767626,"high":1.3984057903,"low":1.3846387863,"close":1.3910005093,"volume":0,"adjclose":1.3910005093},"('GBPUSD=X', datetime.date(2022, 7, 4))":{"open":1.2105804682,"high":1.2165154219,"low":1.1877613068,"close":1.2031002045,"volume":0,"adjclose":1.2031002045},"('GBPUSD=X', datetime.date(2022, 7, 11))":{"open":1.2017786503,"high":1.2019952536,"low":1.1763184071,"close":1.1855996847,"volume":0,"adjclose":1.1855996847},"('GBPUSD=X', datetime.date(2022, 7, 18))":{"open":1.1887921095,"high":1.2064181566,"low":1.1875497103,"close":1.2005999088,"volume":0,"adjclose":1.2005999088},"('GBPUSD=X', datetime.date(2022, 7, 25))":{"open":1.1984229088,"high":1.2242598534,"low":1.1962151527,"close":1.2180001736,"volume":0,"adjclose":1.2180001736},"('GBPUSD=X', datetime.date(2022, 8, 1))":{"open":1.2167818546,"high":1.2291958332,"low":1.2164857388,"close":1.2167373896,"volume":0,"adjclose":1.2167373896},"('GBPUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.2167818546,"high":1.2291958332,"low":1.2164857388,"close":1.2167373896,"volume":0,"adjclose":1.2167373896},"('GC=F', datetime.date(2021, 7, 26))":{"open":1803.6999511719,"high":1832.5999755859,"low":1799.5,"close":1812.5999755859,"volume":243380,"adjclose":1812.5999755859},"('GC=F', datetime.date(2022, 6, 27))":{"open":1830.5,"high":1830.6999511719,"low":1791.5999755859,"close":1798.9000244141,"volume":1046,"adjclose":1798.9000244141},"('GC=F', datetime.date(2022, 7, 4))":{"open":1805.4000244141,"high":1805.4000244141,"low":1732.0999755859,"close":1740.5999755859,"volume":4032,"adjclose":1740.5999755859},"('GC=F', datetime.date(2022, 7, 11))":{"open":1732.5,"high":1736.6999511719,"low":1701.0999755859,"close":1702.4000244141,"volume":2904,"adjclose":1702.4000244141},"('GC=F', datetime.date(2022, 7, 18))":{"open":1712.1999511719,"high":1735,"low":1679.8000488281,"close":1727.0999755859,"volume":2322,"adjclose":1727.0999755859},"('GC=F', datetime.date(2022, 7, 25))":{"open":1727,"high":1765.6999511719,"low":1717.6999511719,"close":1762.9000244141,"volume":185654,"adjclose":1762.9000244141},"('GC=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1727,"high":1765.6999511719,"low":1717.6999511719,"close":1762.9000244141,"volume":185654,"adjclose":1762.9000244141},"('SI=F', datetime.date(2021, 7, 26))":{"open":25.2000007629,"high":25.8549995422,"low":24.6319999695,"close":25.5279998779,"volume":1000,"adjclose":25.5279998779},"('SI=F', datetime.date(2022, 6, 27))":{"open":21.2950000763,"high":21.3299999237,"low":19.2649993896,"close":19.5970001221,"volume":48988,"adjclose":19.5970001221},"('SI=F', datetime.date(2022, 7, 4))":{"open":19.8099994659,"high":20.0699996948,"low":18.8400001526,"close":19.1669998169,"volume":566,"adjclose":19.1669998169},"('SI=F', datetime.date(2022, 7, 11))":{"open":19.1650009155,"high":19.1749992371,"low":18,"close":18.5480003357,"volume":1108,"adjclose":18.5480003357},"('SI=F', datetime.date(2022, 7, 18))":{"open":18.8099994659,"high":18.9400005341,"low":18.1100006104,"close":18.5849990845,"volume":690,"adjclose":18.5849990845},"('SI=F', datetime.date(2022, 7, 25))":{"open":18.4099998474,"high":20.2900009155,"low":18.1849994659,"close":20.1560001373,"volume":980,"adjclose":20.1560001373},"('SI=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":18.4099998474,"high":20.2900009155,"low":18.1849994659,"close":20.1560001373,"volume":980,"adjclose":20.1560001373},"('^DJI', datetime.date(2021, 7, 26))":{"open":35078.8984375,"high":35171.51953125,"low":34871.12890625,"close":34935.46875,"volume":1172870000,"adjclose":34935.46875},"('^DJI', datetime.date(2022, 6, 27))":{"open":31533.599609375,"high":31885.08984375,"low":30431.869140625,"close":31097.259765625,"volume":1634590000,"adjclose":31097.259765625},"('^DJI', datetime.date(2022, 7, 4))":{"open":30903.119140625,"high":31511.4609375,"low":30355.119140625,"close":31338.150390625,"volume":1147620000,"adjclose":31338.150390625},"('^DJI', datetime.date(2022, 7, 11))":{"open":31277.98046875,"high":31367.55078125,"low":30143.9296875,"close":31288.259765625,"volume":1507740000,"adjclose":31288.259765625},"('^DJI', datetime.date(2022, 7, 18))":{"open":31475.98046875,"high":32219.25,"low":30982.970703125,"close":31899.2890625,"volume":1624400000,"adjclose":31899.2890625},"('^DJI', datetime.date(2022, 7, 25))":{"open":31950.9296875,"high":32910.1796875,"low":31705.359375,"close":32845.12890625,"volume":1764330000,"adjclose":32845.12890625},"('^DJI', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":31950.9296875,"high":32910.1796875,"low":31705.359375,"close":32845.12890625,"volume":1764330000,"adjclose":32845.12890625},"('^GSPC', datetime.date(2021, 7, 26))":{"open":4416.3798828125,"high":4429.9702148438,"low":4372.509765625,"close":4395.259765625,"volume":16458580000,"adjclose":4395.259765625},"('^GSPC', datetime.date(2022, 6, 27))":{"open":3920.7600097656,"high":3945.8601074219,"low":3738.669921875,"close":3825.330078125,"volume":21693690000,"adjclose":3825.330078125},"('^GSPC', datetime.date(2022, 7, 4))":{"open":3792.6101074219,"high":3918.5,"low":3742.0600585938,"close":3899.3798828125,"volume":17073700000,"adjclose":3899.3798828125},"('^GSPC', datetime.date(2022, 7, 11))":{"open":3880.9399414062,"high":3880.9399414062,"low":3721.5600585938,"close":3863.1599121094,"volume":19693570000,"adjclose":3863.1599121094},"('^GSPC', datetime.date(2022, 7, 18))":{"open":3883.7900390625,"high":4012.4399414062,"low":3818.6298828125,"close":3961.6298828125,"volume":20385270000,"adjclose":3961.6298828125},"('^GSPC', datetime.date(2022, 7, 25))":{"open":3965.7199707031,"high":4140.1499023438,"low":3910.7399902344,"close":4130.2900390625,"volume":20488830000,"adjclose":4130.2900390625},"('^GSPC', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":3965.7199707031,"high":4140.1499023438,"low":3910.7399902344,"close":4130.2900390625,"volume":20488830000,"adjclose":4130.2900390625},"('^IXIC', datetime.date(2021, 7, 26))":{"open":14807.9501953125,"high":14833.740234375,"low":14503.759765625,"close":14672.6796875,"volume":16171130000,"adjclose":14672.6796875},"('^IXIC', datetime.date(2022, 6, 27))":{"open":11661.01953125,"high":11677.490234375,"low":10850.009765625,"close":11127.849609375,"volume":26652190000,"adjclose":11127.849609375},"('^IXIC', datetime.date(2022, 7, 4))":{"open":10964.1796875,"high":11689.7001953125,"low":10911.4501953125,"close":11635.3095703125,"volume":19116360000,"adjclose":11635.3095703125},"('^IXIC', datetime.date(2022, 7, 11))":{"open":11524.490234375,"high":11541.099609375,"low":11005.9296875,"close":11452.419921875,"volume":21963960000,"adjclose":11452.419921875},"('^IXIC', datetime.date(2022, 7, 18))":{"open":11561.6396484375,"high":12093.01953125,"low":11322.83984375,"close":11834.1103515625,"volume":25230550000,"adjclose":11834.1103515625},"('^IXIC', datetime.date(2022, 7, 25))":{"open":11837.9599609375,"high":12426.259765625,"low":11533.3701171875,"close":12390.6904296875,"volume":23117120000,"adjclose":12390.6904296875},"('^IXIC', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":11837.9599609375,"high":12426.259765625,"low":11533.3701171875,"close":12390.6904296875,"volume":23117120000,"adjclose":12390.6904296875}} From 1215085fbd78e89bc3786c6af29f996a5fa4b220 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 8 Nov 2023 08:02:05 +0000 Subject: [PATCH 13/81] setup.cfg: update dependency to `mmgen-wallet` --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index ce7b703..67288ca 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev4 +3.2.dev5 diff --git a/setup.cfg b/setup.cfg index 339e7f7..3cb1afb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen>=14.0.dev11 + mmgen-wallet>=14.0.dev12 pyyaml yahooquery From b2dc247429927471aa3d402894c2bd692a027684 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 8 Nov 2023 10:31:42 +0000 Subject: [PATCH 14/81] update package name, install instructions in README --- README.md | 8 +++++++- setup.cfg | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 673155e..96807ad 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,13 @@ Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. ## Install: -First, install [MMGen][6]. +### Stable version: + + $ python3 -m pip install --user --upgrade mmgen-node-tools + +### Development version: + +First, install the latest development version of [MMGen][6]. Then, diff --git a/setup.cfg b/setup.cfg index 3cb1afb..ae7676f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = MMGen Node Tools +name = mmgen-node-tools version = file: mmgen_node_tools/data/version description = Optional online tools for the MMGen wallet suite long_description = file: README.md From 554f78690384cf297a9770036a87a701d9e55766 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 9 Nov 2023 05:19:01 +0000 Subject: [PATCH 15/81] update install instructions in README.md --- README.md | 13 +++++++++---- mmgen_node_tools/data/version | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 96807ad..c7ecc29 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,15 @@ Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. ## Install: +If installing as user (without venv), Make sure that `~/.local/bin` is in `PATH`. + ### Stable version: - $ python3 -m pip install --user --upgrade mmgen-node-tools +First, install the [required MMGen packages][7] for your Linux distribution: + +Then, + + $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: @@ -21,9 +27,7 @@ Then, $ git clone https://github.com/mmgen/mmgen-node-tools $ cd mmgen-node-tools $ python3 -m build --no-isolation - $ python3 -m pip install --user dist/*.whl - -Also make sure that `~/.local/bin` is in `PATH`. + $ python3 -m pip install dist/*.whl ## Test: @@ -52,3 +56,4 @@ Donate: 15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w [4]: https://bitcointalk.org/index.php?topic=567069.0 [5]: https://github.com/mmgen/mmgen/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen/ +[7]: https://github.com/mmgen/mmgen/wiki/Install-MMGen-on-Linux diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 67288ca..5d6bce8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev5 +3.2.dev6 From 56327412ca910fd44ba278e9292c31c8160dda36 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Nov 2023 14:45:15 +0000 Subject: [PATCH 16/81] update for mmgen-wallet v14.0.dev22 --- README.md | 14 +++++++------- mmgen_node_tools/Misc.py | 4 ++-- mmgen_node_tools/PeerBlocks.py | 4 ++-- mmgen_node_tools/PollDisplay.py | 4 ++-- mmgen_node_tools/Ticker.py | 4 ++-- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 4 ++-- mmgen_node_tools/main_ticker.py | 4 ++-- setup.cfg | 7 ++++++- test/cmdtest_py_d/ct_main.py | 4 ++-- test/init.sh | 12 ++++++------ .../fakemods/mmgen_node_tools/PeerBlocks.py | 4 ++-- 12 files changed, 36 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index c7ecc29..c95e206 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. ## Install: -If installing as user (without venv), Make sure that `~/.local/bin` is in `PATH`. +If installing as user (without venv), make sure that `~/.local/bin` is in `PATH`. ### Stable version: -First, install the [required MMGen packages][7] for your Linux distribution: +First, install the [required MMGen Wallet packages][7] for your Linux distribution: Then, @@ -20,7 +20,7 @@ Then, ### Development version: -First, install the latest development version of [MMGen][6]. +First, install the latest development version of [MMGen Wallet][6]. Then, @@ -31,7 +31,7 @@ Then, ## Test: -*NOTE: the tests require that the MMGen and MMGen Node Tools repositories be +*NOTE: the tests require that the MMGen Wallet and MMGen Node Tools repositories be located in the same directory.* Initialize the test framework (must be run at least once after cloning, and @@ -54,6 +54,6 @@ Full testing: Donate: 15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w [4]: https://bitcointalk.org/index.php?topic=567069.0 -[5]: https://github.com/mmgen/mmgen/wiki/MMGen-Signing-Keys -[6]: https://github.com/mmgen/mmgen/ -[7]: https://github.com/mmgen/mmgen/wiki/Install-MMGen-on-Linux +[5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys +[6]: https://github.com/mmgen/mmgen-wallet/ +[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux diff --git a/mmgen_node_tools/Misc.py b/mmgen_node_tools/Misc.py index 999b68b..5aa27f5 100755 --- a/mmgen_node_tools/Misc.py +++ b/mmgen_node_tools/Misc.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmgen_node_tools.Misc: miscellaneous data and functions for the MMGen Node Tools suite diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index 0b386fb..c3d7c68 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 5bbfecd..72bedac 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ mmgen_node_tools.PollDisplay: update and display RPC data; get input from user diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6c15906..5a6a6a2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 5d6bce8..2c19032 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev6 +3.2.dev7 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 34c6236..7af2b77 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index eafa34a..f758089 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen https://github.com/mmgen/mmgen-node-tools -# https://gitlab.com/mmgen/mmgen https://gitlab.com/mmgen/mmgen-node-tools +# https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools """ mmnode-ticker: Display price information for cryptocurrency and other assets diff --git a/setup.cfg b/setup.cfg index ae7676f..4cb6a92 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,19 +11,24 @@ license = GNU GPL v3 platforms = Linux, Armbian, Raspbian, MS Windows keywords = file: mmgen_node_tools/data/keywords project_urls = + Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = Programming Language :: Python :: 3 License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows + Environment :: Console + Topic :: Office/Business :: Financial + Topic :: Security :: Cryptography + Development Status :: 5 - Production/Stable [options] python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.dev12 + mmgen-wallet>=14.0.dev22 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index ad1ce41..5f5c691 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite diff --git a/test/init.sh b/test/init.sh index 913ac9c..769490a 100755 --- a/test/init.sh +++ b/test/init.sh @@ -31,22 +31,22 @@ done shift $((OPTIND-1)) -mm_repo='../mmgen' +wallet_repo='../mmgen-wallet' die() { echo -e ${YELLOW}ERROR: $1$RESET; false; } becho() { echo -e $BLUE$1$RESET; } check_mmgen_repo() { - ( cd $mm_repo; python3 ./setup.py --url | grep -iq 'mmgen' ) + ( cd $wallet_repo; python3 ./setup.py --url | grep -iq 'mmgen' ) } build_mmgen_extmod() { - ( cd $mm_repo; python3 ./setup.py build_ext --inplace ) + ( cd $wallet_repo; python3 ./setup.py build_ext --inplace ) } create_dir_links() { for link_name in 'mmgen' 'scripts'; do - target="$mm_repo/$link_name" + target="$wallet_repo/$link_name" if [ -e $link_name ]; then [ $(realpath --relative-to=. $link_name) == $target ] || die "'$link_name' does not point to '$target'" else @@ -73,7 +73,7 @@ create_test_links() { [ "$path" ] || continue pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) - target="$mm_repo/$path" + target="$wallet_repo/$path" if [ ! -e "$target" ]; then echo "Target path $target is missing! Cannot proceed" exit 1 @@ -101,7 +101,7 @@ set -e becho 'Initializing MMGen Node Tools Test Suite' -check_mmgen_repo || die "MMGen repository not found at $mm_repo!" +check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py index 2484628..7c2fdf3 100644 --- a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -5,8 +5,8 @@ # Licensed under the GNU General Public License, Version 3: # https://www.gnu.org/licenses # Public project repositories: -# https://github.com/mmgen/mmgen -# https://gitlab.com/mmgen/mmgen +# https://github.com/mmgen/mmgen-wallet +# https://gitlab.com/mmgen/mmgen-wallet """ fakemods.mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes - test data From d0874f9a2d6475d81e50f6809aca6a05c4dced88 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 21 Nov 2023 15:30:42 +0000 Subject: [PATCH 17/81] test/init.sh: handle broken dir symlinks --- mmgen_node_tools/data/version | 2 +- test/cmdtest_py_d/cfg.py | 5 +++-- test/init.sh | 12 +++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 2c19032..21ca9e7 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev7 +3.2.dev8 diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_py_d/cfg.py index 5aacbf9..96bfe9f 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -12,7 +12,7 @@ test.cmdtest_py_d.cfg: configuration data for cmdtest.py """ -import os +cmd_groups_altcoin = [] cmd_groups_dfl = { 'main': ('CmdTestMain',{}), @@ -29,4 +29,5 @@ cfgs = { '3': {}, # main } -def fixup_cfgs(): pass +def fixup_cfgs(): + pass diff --git a/test/init.sh b/test/init.sh index 769490a..81727bb 100755 --- a/test/init.sh +++ b/test/init.sh @@ -47,9 +47,15 @@ build_mmgen_extmod() { create_dir_links() { for link_name in 'mmgen' 'scripts'; do target="$wallet_repo/$link_name" - if [ -e $link_name ]; then - [ $(realpath --relative-to=. $link_name) == $target ] || die "'$link_name' does not point to '$target'" - else + if [ -L $link_name ]; then + [ "$(realpath --relative-to=. $link_name 2>/dev/null)" == $target ] || { + echo "Removing broken symlink '$link_name'" + rm $link_name + } + elif [ -e $link_name ]; then + die "'$link_name' is not a symbolic link. Please remove or relocate it and re-run this script" + fi + if [ ! -e $link_name ]; then echo "Creating symlink: $link_name" ln -s $target fi From 55fdcb29f7ef476676cb8194dcdab690b08e3b46 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 27 Nov 2023 09:18:43 +0000 Subject: [PATCH 18/81] Support Windows/MSYS2 --- README.md | 82 ++++++++++++++++++++++++++------- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_main.py | 6 ++- test/cmdtest_py_d/ct_regtest.py | 4 +- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c95e206..d3b18ba 100644 --- a/README.md +++ b/README.md @@ -4,36 +4,66 @@ Requires modules from the [MMGen online/offline cryptocurrency wallet][6]. -Currently tested on Linux only. Some scripts may not work under Windows/MSYS2. - ## Install: If installing as user (without venv), make sure that `~/.local/bin` is in `PATH`. +#### Windows/MSYS2: + +> Install [MSYS2 and the MMGen Wallet dependencies][8], skipping installation of +> scrypt, libsecp256k1 and the wallet itself if desired. + +> Install some additional dependencies: + +```bash + $ pacman -S \ + mingw-w64-ucrt-x86_64-python-pandas \ + mingw-w64-ucrt-x86_64-python-tqdm \ + mingw-w64-ucrt-x86_64-python-lxml + $ python3 -m pip install requests-futures + $ python3 -m pip install --no-deps yahooquery +``` + +#### Linux: + +> Install the [required MMGen Wallet packages][7] for your Linux distribution. + ### Stable version: -First, install the [required MMGen Wallet packages][7] for your Linux distribution: - -Then, - - $ python3 -m pip install --upgrade mmgen-node-tools +```bash +$ python3 -m pip install --upgrade mmgen-node-tools +``` ### Development version: -First, install the latest development version of [MMGen Wallet][6]. +Install the latest development version of [MMGen Wallet][6] for your platform. -Then, - - $ git clone https://github.com/mmgen/mmgen-node-tools - $ cd mmgen-node-tools - $ python3 -m build --no-isolation - $ python3 -m pip install dist/*.whl +```bash +$ git clone https://github.com/mmgen/mmgen-node-tools +$ cd mmgen-node-tools +$ python3 -m build --no-isolation +$ python3 -m pip install dist/*.whl +``` ## Test: *NOTE: the tests require that the MMGen Wallet and MMGen Node Tools repositories be located in the same directory.* +#### Windows/MSYS2: + +> *Tested only on NTFS – with ReFS your mileage may vary* + +> Turn on Developer Mode to enable symlinks: +``` + Settings -> Update & Security -> For developers -> Developer Mode: On +``` +> and add this to your `~/.bashrc`: +```bash + export MSYS=winsymlinks:nativestrict +``` +> Close and reopen the MSYS2 terminal to update your environment. + Initialize the test framework (must be run at least once after cloning, and possibly again after a pull if tests have been updated): @@ -49,11 +79,29 @@ Full testing: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -[**Forum**][4] | -[PGP Public Key][5] | -Donate: 15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w +Homepage: +[Clearnet](https://mmgen-wallet.cc) | +[I2P](http://mmgen-wallet.i2p) | +[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion) +Code repository: +[Clearnet](https://mmgen.org/project/mmgen/mmgen-wallet) | +[I2P](http://mmgen-wallet.i2p/project/mmgen/mmgen-wallet) | +[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion/project/mmgen/mmgen-wallet) +Code repository mirrors: +[Github](https://github.com/mmgen/mmgen-wallet) | +[Gitlab](https://gitlab.com/mmgen/mmgen-wallet) | +[Gitflic](https://gitflic.ru/project/mmgen/mmgen-wallet) +[Keybase](https://keybase.io/mmgen) | +[Reddit](https://www.reddit.com/user/mmgen-py) | +[Bitcointalk](https://bitcointalk.org/index.php?topic=567069.new#new) +[PGP Signing Key][5]: 5C84 CB45 AEE2 250F 31A6 A570 3F8B 1861 E32B 7DA2 +Donate: + ⊙ BTC: *bc1qxmymxf8p5ckvlxkmkwgw8ap5t2xuaffmrpexap* + ⊙ BCH: *15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w* + ⊙ XMR: *8B14zb8wgLuKDdse5p8f3aKpFqRdB4i4xj83b7BHYABHMvHifWxiDXeKRELnaxL5FySfeRRS5girgUvgy8fQKsYMEzPUJ8h* [4]: https://bitcointalk.org/index.php?topic=567069.0 [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ [7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux +[8]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Microsoft-Windows#a_m diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 21ca9e7..a91fcc8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev8 +3.2.dev9 diff --git a/setup.cfg b/setup.cfg index 4cb6a92..62f7074 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.dev22 + mmgen-wallet>=14.0.dev25 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index 5f5c691..7c38bcf 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -35,7 +35,7 @@ class CmdTestMain(CmdTestBase): "'mmnode-peerblocks' script", ('peerblocks1', '--help'), ('peerblocks2', 'interactive (popen spawn)'), - ('peerblocks3', 'interactive, 80 columns (pexpect_spawn)'), + ('peerblocks3', 'interactive, 80 columns (pexpect_spawn [on Linux])'), ), } @@ -94,4 +94,6 @@ class CmdTestMain(CmdTestBase): return t def peerblocks3(self): - return self.peerblocks2(['--columns=80'],pexpect_spawn=True) + return self.peerblocks2( + ['--columns=80'], + pexpect_spawn = sys.platform != 'win32' ) diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index b17b4c6..e554930 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -12,7 +12,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ -import os +import sys,os from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto @@ -134,6 +134,8 @@ class CmdTestRegtest(CmdTestBase): def netrate2(self): t = self.netrate( [], r'sent:.*' ) t.kill(15) + if sys.platform == 'win32': + return 'ok' t.req_exit_val = -15 return t From 4b22d67e37d22a73701ff7b90236389b448bda45 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 27 Nov 2023 09:32:24 +0000 Subject: [PATCH 19/81] update README.md --- README.md | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d3b18ba..4d3fe28 100644 --- a/README.md +++ b/README.md @@ -12,17 +12,16 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > Install [MSYS2 and the MMGen Wallet dependencies][8], skipping installation of > scrypt, libsecp256k1 and the wallet itself if desired. - +> > Install some additional dependencies: - -```bash - $ pacman -S \ - mingw-w64-ucrt-x86_64-python-pandas \ - mingw-w64-ucrt-x86_64-python-tqdm \ - mingw-w64-ucrt-x86_64-python-lxml - $ python3 -m pip install requests-futures - $ python3 -m pip install --no-deps yahooquery -``` +> ```bash +> $ pacman -S \ +> mingw-w64-ucrt-x86_64-python-pandas \ +> mingw-w64-ucrt-x86_64-python-tqdm \ +> mingw-w64-ucrt-x86_64-python-lxml +> $ python3 -m pip install requests-futures +> $ python3 -m pip install --no-deps yahooquery +> ``` #### Linux: @@ -53,15 +52,15 @@ located in the same directory.* #### Windows/MSYS2: > *Tested only on NTFS – with ReFS your mileage may vary* - +> > Turn on Developer Mode to enable symlinks: -``` - Settings -> Update & Security -> For developers -> Developer Mode: On -``` +> ``` +> Settings -> Update & Security -> For developers -> Developer Mode: On +> ``` > and add this to your `~/.bashrc`: -```bash - export MSYS=winsymlinks:nativestrict -``` +> ```bash +> export MSYS=winsymlinks:nativestrict +> ``` > Close and reopen the MSYS2 terminal to update your environment. Initialize the test framework (must be run at least once after cloning, and From ef0acc254225d712465cfca9558b30bbd628f41c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 27 Nov 2023 10:55:22 +0000 Subject: [PATCH 20/81] Version 3.2.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index a91fcc8..944880f 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev9 +3.2.0 diff --git a/setup.cfg b/setup.cfg index 62f7074..f737c62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.dev25 + mmgen-wallet==14.0.0 pyyaml yahooquery From 3dace6b188aaed1909f9bc4a6973df996efaecaa Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 30 Nov 2023 09:19:10 +0000 Subject: [PATCH 21/81] README.md: fix MSWin deps, fix repo URLs, add Twitter link --- README.md | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4d3fe28..f72051f 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,7 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > > Install some additional dependencies: > ```bash -> $ pacman -S \ -> mingw-w64-ucrt-x86_64-python-pandas \ -> mingw-w64-ucrt-x86_64-python-tqdm \ -> mingw-w64-ucrt-x86_64-python-lxml +> $ pacman -S mingw-w64-ucrt-x86_64-python-pandas > $ python3 -m pip install requests-futures > $ python3 -m pip install --no-deps yahooquery > ``` @@ -35,7 +32,7 @@ $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: -Install the latest development version of [MMGen Wallet][6] for your platform. +Install the latest development version of [MMGen Wallet][6] for your platform: ```bash $ git clone https://github.com/mmgen/mmgen-node-tools @@ -65,32 +62,34 @@ located in the same directory.* Initialize the test framework (must be run at least once after cloning, and possibly again after a pull if tests have been updated): - - $ test/init.sh - +``` +$ test/init.sh +``` BTC-only testing: - - $ test/test-release.sh -A - +``` +$ test/test-release.sh -A +``` Full testing: - - $ test/test-release.sh +``` +$ test/test-release.sh +``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Homepage: -[Clearnet](https://mmgen-wallet.cc) | +[Clearnet](https://mmgen.org) | [I2P](http://mmgen-wallet.i2p) | [Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion) Code repository: -[Clearnet](https://mmgen.org/project/mmgen/mmgen-wallet) | -[I2P](http://mmgen-wallet.i2p/project/mmgen/mmgen-wallet) | -[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion/project/mmgen/mmgen-wallet) +[Clearnet](https://mmgen.org/project/mmgen/mmgen-node-tools) | +[I2P](http://mmgen-wallet.i2p/project/mmgen/mmgen-node-tools) | +[Onion](http://mmgen55rtcahqfp2hn3v7syqv2wqanks5oeezqg3ykwfkebmouzjxlad.onion/project/mmgen/mmgen-node-tools) Code repository mirrors: -[Github](https://github.com/mmgen/mmgen-wallet) | -[Gitlab](https://gitlab.com/mmgen/mmgen-wallet) | -[Gitflic](https://gitflic.ru/project/mmgen/mmgen-wallet) +[Github](https://github.com/mmgen/mmgen-node-tools) | +[Gitlab](https://gitlab.com/mmgen/mmgen-node-tools) | +[Gitflic](https://gitflic.ru/project/mmgen/mmgen-node-tools) [Keybase](https://keybase.io/mmgen) | +[Twitter](https://twitter.com/TheMMGenProject) | [Reddit](https://www.reddit.com/user/mmgen-py) | [Bitcointalk](https://bitcointalk.org/index.php?topic=567069.new#new) [PGP Signing Key][5]: 5C84 CB45 AEE2 250F 31A6 A570 3F8B 1861 E32B 7DA2 @@ -99,7 +98,6 @@ Donate:  ⊙ BCH: *15TLdmi5NYLdqmtCqczUs5pBPkJDXRs83w*  ⊙ XMR: *8B14zb8wgLuKDdse5p8f3aKpFqRdB4i4xj83b7BHYABHMvHifWxiDXeKRELnaxL5FySfeRRS5girgUvgy8fQKsYMEzPUJ8h* -[4]: https://bitcointalk.org/index.php?topic=567069.0 [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ [7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux From de29b213ff824ee247017c16c52a48378f1c5e17 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 30 Nov 2023 11:25:34 +0000 Subject: [PATCH 22/81] bump version and wallet dependency --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 944880f..0e5a99a 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.0 +3.3.dev0 diff --git a/setup.cfg b/setup.cfg index f737c62..7827398 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet==14.0.0 + mmgen-wallet>=14.0.0 pyyaml yahooquery From cc0f4729c6efa8db40e93b43d88f569cd9442c03 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 12 Dec 2023 09:58:01 +0000 Subject: [PATCH 23/81] update for mmgen-wallet v14.1.dev2 --- cmds/mmnode-addrbal | 2 +- cmds/mmnode-blocks-info | 2 +- cmds/mmnode-feeview | 2 +- cmds/mmnode-halving-calculator | 2 +- cmds/mmnode-netrate | 2 +- cmds/mmnode-peerblocks | 2 +- cmds/mmnode-ticker | 2 +- cmds/mmnode-txfind | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/unit_tests_d/ut_dep.py | 21 +++++++++++++++++++++ 11 files changed, 31 insertions(+), 10 deletions(-) create mode 100755 test/unit_tests_d/ut_dep.py diff --git a/cmds/mmnode-addrbal b/cmds/mmnode-addrbal index 3a5bc68..a14edc0 100755 --- a/cmds/mmnode-addrbal +++ b/cmds/mmnode-addrbal @@ -14,4 +14,4 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain from mmgen.main import launch -launch('addrbal',package='mmgen_node_tools') +launch(mod='addrbal',package='mmgen_node_tools') diff --git a/cmds/mmnode-blocks-info b/cmds/mmnode-blocks-info index a1c855e..d542d1a 100755 --- a/cmds/mmnode-blocks-info +++ b/cmds/mmnode-blocks-info @@ -14,4 +14,4 @@ mmnode-blocks-info: Display information about a block or range of blocks from mmgen.main import launch -launch('blocks_info',package='mmgen_node_tools') +launch(mod='blocks_info',package='mmgen_node_tools') diff --git a/cmds/mmnode-feeview b/cmds/mmnode-feeview index 5a4625e..8b639bb 100755 --- a/cmds/mmnode-feeview +++ b/cmds/mmnode-feeview @@ -14,4 +14,4 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool from mmgen.main import launch -launch('feeview',package='mmgen_node_tools') +launch(mod='feeview',package='mmgen_node_tools') diff --git a/cmds/mmnode-halving-calculator b/cmds/mmnode-halving-calculator index ebf1f6d..1759f77 100755 --- a/cmds/mmnode-halving-calculator +++ b/cmds/mmnode-halving-calculator @@ -14,4 +14,4 @@ mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) from mmgen.main import launch -launch('halving_calculator',package='mmgen_node_tools') +launch(mod='halving_calculator',package='mmgen_node_tools') diff --git a/cmds/mmnode-netrate b/cmds/mmnode-netrate index cfefab2..f8c4ad1 100755 --- a/cmds/mmnode-netrate +++ b/cmds/mmnode-netrate @@ -14,4 +14,4 @@ mmnode-netrate: Bitcoin daemon network rate monitor from mmgen.main import launch -launch('netrate',package='mmgen_node_tools') +launch(mod='netrate',package='mmgen_node_tools') diff --git a/cmds/mmnode-peerblocks b/cmds/mmnode-peerblocks index be4bb41..5a02741 100755 --- a/cmds/mmnode-peerblocks +++ b/cmds/mmnode-peerblocks @@ -14,4 +14,4 @@ mmnode-peerblocks: List blocks in flight, disconnect stalling nodes from mmgen.main import launch -launch('peerblocks',package='mmgen_node_tools') +launch(mod='peerblocks',package='mmgen_node_tools') diff --git a/cmds/mmnode-ticker b/cmds/mmnode-ticker index d6e91fb..f0160e7 100755 --- a/cmds/mmnode-ticker +++ b/cmds/mmnode-ticker @@ -14,4 +14,4 @@ mmnode-ticker: Display price information for cryptocurrency and other assets from mmgen.main import launch -launch('ticker',package='mmgen_node_tools') +launch(mod='ticker',package='mmgen_node_tools') diff --git a/cmds/mmnode-txfind b/cmds/mmnode-txfind index 05213a8..fd48cf4 100755 --- a/cmds/mmnode-txfind +++ b/cmds/mmnode-txfind @@ -14,4 +14,4 @@ mmnode-txfind: Find a transaction in the blockchain or mempool from mmgen.main import launch -launch('txfind',package='mmgen_node_tools') +launch(mod='txfind',package='mmgen_node_tools') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0e5a99a..f5f664c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev0 +3.3.dev1 diff --git a/setup.cfg b/setup.cfg index 7827398..c99a9ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen-wallet>=14.0.0 + mmgen-wallet>=14.1.dev2 pyyaml yahooquery diff --git a/test/unit_tests_d/ut_dep.py b/test/unit_tests_d/ut_dep.py new file mode 100755 index 0000000..641c3c3 --- /dev/null +++ b/test/unit_tests_d/ut_dep.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +""" +test.unit_tests_d.ut_dep: dependency unit tests for the MMGen Node Tools + + Test whether dependencies are installed and functional. + No data verification is performed. +""" + +from ..include.common import vmsg,imsg +from mmgen.color import yellow + +class unit_tests: + + def yahooquery(self,name,ut): + try: + from yahooquery import Ticker + return True + except ImportError: + imsg(yellow('Unable to import Ticker from yahooquery')) + return False From 6f2925f7973b9918a5582c35167bcf9ffe09b92c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 10 Feb 2024 15:15:18 +0000 Subject: [PATCH 24/81] test/init.sh, MANIFEST.in: fixes, cleanups --- MANIFEST.in | 10 ++++++++-- mmgen_node_tools/Ticker.py | 4 ++-- test/init.sh | 31 +++++++++++++++++++++---------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 7bdab90..370da07 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,10 @@ include README.md LICENSE -include test/test-release.sh -include test/unit_tests_d/*.py + include mmgen_node_tools/data/* + +include test/init.sh +include test/test-release.d/*.sh +include test/unit_tests_d/*.py +include test/cmdtest_py_d/*.py +include test/overlay/fakemods/mmgen_node_tools/*.py +include test/ref/*/* diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 5a6a6a2..e8b704f 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -193,7 +193,7 @@ class DataSource: @property def timeout(self): - return 5 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit + return 0 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit def json_data_error_msg(self,json_text): tor_captcha_msg = f""" @@ -265,7 +265,7 @@ class DataSource: @property def timeout(self): - return 5 if gcfg.test_suite else self.ratelimit + return 0 if gcfg.test_suite else self.ratelimit @property def symbols(self): diff --git a/test/init.sh b/test/init.sh index 81727bb..51e4e9e 100755 --- a/test/init.sh +++ b/test/init.sh @@ -10,6 +10,7 @@ RED="\e[31;1m" GREEN="\e[32;1m" YELLOW="\e[33;1m" BLUE="\e[34;1m" RESET="\e[0m" +set -e set -o errtrace set -o functrace @@ -17,14 +18,19 @@ trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT trap 'echo -e "${RED}Node Tools test suite initialization exited with error (line $BASH_LINENO) $RESET"' ERR umask 0022 +STDOUT_DEVNULL='>/dev/null' +STDERR_DEVNULL='2>/dev/null' + PROGNAME=$(basename $0) -while getopts h OPT +while getopts hv OPT do case "$OPT" in h) printf " %-16s Initialize the MMGen Node Tools test suite\n" "${PROGNAME}:" echo " USAGE: $PROGNAME" echo " OPTIONS: '-h' Print this help message" + echo " -v Be more verbose" exit ;; + v) VERBOSE=1 STDOUT_DEVNULL='' STDERR_DEVNULL='' ;; *) exit ;; esac done @@ -41,7 +47,10 @@ check_mmgen_repo() { } build_mmgen_extmod() { - ( cd $wallet_repo; python3 ./setup.py build_ext --inplace ) + ( + cd $wallet_repo + eval "python3 ./setup.py build_ext --inplace $STDOUT_DEVNULL $STDERR_DEVNULL" + ) } create_dir_links() { @@ -49,14 +58,14 @@ create_dir_links() { target="$wallet_repo/$link_name" if [ -L $link_name ]; then [ "$(realpath --relative-to=. $link_name 2>/dev/null)" == $target ] || { - echo "Removing broken symlink '$link_name'" + [ "$VERBOSE" ] && echo "Removing broken symlink '$link_name'" rm $link_name } elif [ -e $link_name ]; then die "'$link_name' is not a symbolic link. Please remove or relocate it and re-run this script" fi if [ ! -e $link_name ]; then - echo "Creating symlink: $link_name" + [ "$VERBOSE" ] && echo "Creating symlink: $link_name" ln -s $target fi done @@ -87,32 +96,34 @@ create_test_links() { fs="%-8s %-16s %s -> %s\n" if [ $type == 'hard' ]; then if [ -L $path ]; then - printf "$fs" "Deleting" "symbolic link:" $path $target + [ "$VERBOSE" ] && printf "$fs" "Deleting" "symbolic link:" $path $target rm -rf $path elif [ -e $path ]; then if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then - printf "$fs" "Deleting" "stale hard link:" $path "?" + [ "$VERBOSE" ] && printf "$fs" "Deleting" "stale hard link:" $path "?" rm -rf $path fi fi fi if [ ! -e $path ]; then # link is either absent or a broken symlink - printf "$fs" "Creating" "$type link:" $path $target + [ "$VERBOSE" ] && printf "$fs" "Creating" "$type link:" $path $target ( cd "$(dirname $path)" && ln -f $symlink_arg $pfx$target ) fi done <<<$paths } -set -e - becho 'Initializing MMGen Node Tools Test Suite' check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod +[ "$VERBOSE" ] && becho 'Creating links to mmgen-wallet repo' + create_dir_links create_test_links -becho 'OK' +[ "$VERBOSE" ] && becho 'OK' + +true From 7c7c2c7da839ce3139c2260eb293d0e67fa1acf7 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 10 Feb 2024 15:15:18 +0000 Subject: [PATCH 25/81] mmnode-ticker: backport fix for Python 3.9 --- mmgen_node_tools/Ticker.py | 3 ++- mmgen_node_tools/data/version | 2 +- setup.cfg | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index e8b704f..33fb1fa 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -440,7 +440,8 @@ def gen_data(data): d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] - d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp()) + # .replace('Z','+00:00') -- Python 3.9 backport + d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated'].replace('Z','+00:00')).timestamp()) yield (d['id'],d) found[k].add(d[k]) wants[k].remove(d[k]) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f5f664c..0ed23d0 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev1 +3.3.dev2 diff --git a/setup.cfg b/setup.cfg index c99a9ef..82704a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,11 +24,11 @@ classifiers = Development Status :: 5 - Production/Stable [options] -python_requires = >=3.8 +python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev2 + mmgen-wallet>=14.1.dev8 pyyaml yahooquery From db4ba26dced535ac60465a71c7ff939305b3ae6a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 8 Mar 2024 15:32:54 +0000 Subject: [PATCH 26/81] update testing for MMGen Wallet v14.1.dev19 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 13 ++++++------- test/cmdtest_py_d/ct_regtest.py | 26 +++++++++++++++++--------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0ed23d0..3170c6e 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev2 +3.3.dev3 diff --git a/setup.cfg b/setup.cfg index 82704a4..f0c6629 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev8 + mmgen-wallet>=14.1.dev19 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 3dbac50..b4b83e9 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -106,10 +106,11 @@ class CmdTestScripts(CmdTestBase): shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' - def ticker(self,args=[],expect_list=None,cached=True): + def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): t = self.spawn( f'mmnode-ticker', - (['--cached-data'] if cached else []) + self.ticker_args + args ) + (['--cached-data'] if cached else []) + self.ticker_args + args, + exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) return t @@ -120,12 +121,11 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False) + t = self.ticker(cached=False, exit_val=3) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - ret = t.expect(['proxy host could not be resolved','ProxyError']) - t.req_exit_val = 3 if ret == 0 else 1 + t.expect(['proxy host could not be resolved', 'ProxyError']) return t def ticker3(self): @@ -165,9 +165,8 @@ class CmdTestScripts(CmdTestBase): return t def ticker6(self): - t = self.ticker( ['--wide','--portfolio'], None ) + t = self.ticker(['--wide','--portfolio'], None, exit_val=1) t.expect('No portfolio') - t.req_exit_val = 1 return t def ticker7(self): # demo diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index e554930..c89f8fa 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -39,6 +39,8 @@ class CmdTestRegtest(CmdTestBase): tmpdir_nums = [1] color = True deterministic = False + bdb_wallet = True + cmd_group_in = ( ('setup', 'regtest mode setup'), ('subgroup.netrate', []), @@ -111,20 +113,27 @@ class CmdTestRegtest(CmdTestBase): die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) - self.regtest = MMGenRegtest(cfg,self.proto.coin) + + self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' + self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) from shutil import rmtree - try: rmtree(joinpath(self.tr.data_dir,'regtest')) - except: pass - t = self.spawn('mmgen-regtest',['-n','setup']) + try: + rmtree(joinpath(self.tr.data_dir,'regtest')) + except: + pass + t = self.spawn( + 'mmgen-regtest', + (['--bdb-wallet'] if self.use_bdb_wallet else []) + + ['--setup-no-stop-daemon', 'setup']) for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): t.expect(s) return t - def netrate(self,add_args,expect_str): - t = self.spawn( 'mmnode-netrate', args1 + add_args ) + def netrate(self, add_args, expect_str, exit_val=None): + t = self.spawn('mmnode-netrate', args1 + add_args, exit_val=exit_val) t.expect(expect_str,regex=True) return t @@ -132,11 +141,10 @@ class CmdTestRegtest(CmdTestBase): return self.netrate( ['--help'], 'USAGE:.*' ) def netrate2(self): - t = self.netrate( [], r'sent:.*' ) + t = self.netrate([], r'sent:.*', exit_val=-15) t.kill(15) if sys.platform == 'win32': return 'ok' - t.req_exit_val = -15 return t def halving_calculator(self,add_args,expect_list): @@ -326,7 +334,7 @@ class CmdTestRegtest(CmdTestBase): return await do_tx( [{ 'txid': tx_input['txid'], 'vout': 0 }], outputs, - r.miner_wif ) + await r.miner_wif) async def do_tx2(tx,pairno): fee = fees[pairno] From c4ace71049ca48e00bdd0a6d204df96ddf71526c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 10 Mar 2024 14:44:57 +0000 Subject: [PATCH 27/81] minor fixes and cleanups --- mmgen_node_tools/Ticker.py | 16 ++++++++++++---- test/cmdtest_py_d/ct_misc.py | 5 +++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 33fb1fa..806b0e9 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -26,6 +26,7 @@ from collections import namedtuple from mmgen.color import red,yellow,green,blue,orange,gray from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.ui import do_pager homedir = os.getenv('HOME') dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') @@ -407,6 +408,8 @@ def gen_data(data): id = get_id(k,v) if wants['id']: if id in wants['id']: + if not isinstance(v,dict): + die(2, str(v)) if id in found['id']: die(1,dup_sym_errmsg(id)) if m := data['hi'].get(k): @@ -517,7 +520,6 @@ def main(): return if gcfg.list_ids: - from mmgen.ui import do_pager do_pager('\n'.join(e['id'] for e in src_data['cc'])) return @@ -628,9 +630,10 @@ def make_cfg(gcfg_arg): def get_portfolio(): return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'} - def parse_add_precision(s): - if not s: + def parse_add_precision(arg): + if not arg: return 0 + s = str(arg) if not (s.isdigit() and s.isascii()): die(1,f'{s}: invalid parameter for --add-precision (not an integer)') if int(s) > 30: @@ -706,7 +709,12 @@ def make_cfg(gcfg_arg): cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None, + portfolio = + get_portfolio() + if cfg_in.portfolio + and gcfg.portfolio + and not query + else None, percent_cols = parse_percent_cols(gcfg.percent_cols) ) diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index b4b83e9..8127e1d 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -121,11 +121,12 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False, exit_val=3) + t = self.ticker(cached=False) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - t.expect(['proxy host could not be resolved', 'ProxyError']) + ret = t.expect(['proxy host could not be resolved', 'ProxyError']) + t.exit_val = 1 if ret else 3 return t def ticker3(self): From 867026e8edd370dbbf655d2ea20ad384a50d4f4e Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 10 Mar 2024 14:44:57 +0000 Subject: [PATCH 28/81] mmnode-ticker: add config file vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following command-line opts are now configurable in ‘ticker-cfg.yaml’: add_precision asset_limit btc cached_data elapsed name_labels pager percent_cols thousands_comma update_time verbose --- mmgen_node_tools/Ticker.py | 65 ++++++++++++++++++++++------------- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 806b0e9..fd05f99 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red,yellow,green,blue,orange,gray -from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.util import msg,msg_r,Msg,Msg_r,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -75,7 +75,7 @@ class DataSource: ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], ['--compressed'], # adds 'Accept-Encoding: gzip' ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], - ['--silent', not gcfg.verbose], + ['--silent', not cfg.verbose], [self.api_url] ) @@ -101,7 +101,9 @@ class DataSource: if not os.path.exists(self.json_fn): open(self.json_fn,'w').write('{}') - if gcfg.cached_data: + use_cached_data = cfg.cached_data and not gcfg.download + + if use_cached_data: data_type = 'json' data_in = open(self.json_fn).read() else: @@ -112,8 +114,8 @@ class DataSource: msg('') self.fetch_delay() msg_r(f'Fetching {self.data_desc} from {self.api_host}...') - if self.has_verbose: - gcfg._util.vmsg('') + if self.has_verbose and cfg.verbose: + msg('') data_in = self.get_data_from_network() msg('done') if gcfg.testing: @@ -133,14 +135,14 @@ class DataSource: json_text = json.dumps(data_in) if not data: - if gcfg.cached_data: + if use_cached_data: die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') else: die(2,'Remote host returned no data!') elif 'error' in data: die(1,data['error']) - if gcfg.cached_data: + if use_cached_data: msg(f'Using cached data from ~/{self.json_fn_rel}') else: open(self.json_fn,'w').write(json_text) @@ -171,7 +173,7 @@ class DataSource: dfl_asset_limit = 2000 def __init__(self): - self.asset_limit = int(gcfg.asset_limit or self.dfl_asset_limit) + self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self,elapsed): return ( @@ -528,9 +530,8 @@ def main(): data = dict(gen_data(src_data)) - gcfg._util.stdout_or_pager( - '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n' - ) + (do_pager if cfg.pager else Msg_r)( + '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n') def make_cfg(gcfg_arg): @@ -670,7 +671,15 @@ def make_cfg(gcfg_arg): 'proxy', 'proxy2', 'portfolio', - 'percent_cols' ]) + 'percent_cols', + 'asset_limit', + 'cached_data', + 'elapsed', + 'name_labels', + 'pager', + 'thousands_comma', + 'update_time', + 'verbose']) global gcfg,cfg_in,src_cls,cfg @@ -704,18 +713,26 @@ def make_cfg(gcfg_arg): query = query, adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ), clsname = 'trading' if query else 'overview', - btc_only = gcfg.btc, - add_prec = parse_add_precision(gcfg.add_precision), + btc_only = gcfg.btc or cfg_in.cfg.get('btc'), + add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = get_portfolio() if cfg_in.portfolio - and gcfg.portfolio + and (gcfg.portfolio or cfg_in.cfg.get('portfolio')) and not query else None, - percent_cols = parse_percent_cols(gcfg.percent_cols) + percent_cols = parse_percent_cols(gcfg.percent_cols or cfg_in.cfg.get('percent_cols')), + asset_limit = gcfg.asset_limit or cfg_in.cfg.get('asset_limit'), + cached_data = gcfg.cached_data or cfg_in.cfg.get('cached_data'), + elapsed = gcfg.elapsed or cfg_in.cfg.get('elapsed'), + name_labels = gcfg.name_labels or cfg_in.cfg.get('name_labels'), + pager = gcfg.pager or cfg_in.cfg.get('pager'), + thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), + update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), + verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), ) def get_cfg_in(): @@ -753,10 +770,10 @@ class Ticker: def __init__(self,data): - self.comma = ',' if gcfg.thousands_comma else '' + self.comma = ',' if cfg.thousands_comma else '' self.col1_wid = max(len('TOTAL'),( - max(len(self.create_label(d['id'])) for d in data.values()) if gcfg.name_labels else + max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else max(len(d['symbol']) for d in data.values()) )) + 1 @@ -769,7 +786,7 @@ class Ticker: def format_last_update_col(self,cross_assets=()): - if gcfg.elapsed: + if cfg.elapsed: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: @@ -918,7 +935,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( - lbl = (self.create_label(d['id']) if gcfg.name_labels else d['symbol']), + lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), pc3 = fmt_pct(d.get('percent_change_1y')), @@ -969,11 +986,11 @@ class Ticker: ( 'pct1m', 'm' in cfg.percent_cols ), ( 'pct1w', 'w' in cfg.percent_cols ), ( 'pct1d', 'd' in cfg.percent_cols ), - ( 'update_time', gcfg.update_time ), + ( 'update_time', cfg.update_time ), ) if b] ) cols2 = list(cols) - if gcfg.update_time: + if cfg.update_time: cols2.pop() cols2.append('amt') @@ -1058,7 +1075,7 @@ class Ticker: if self.show_adj: self.fs_str += ' {p_adj}' self.hl_wid += self.max_wid + 1 - if gcfg.update_time: + if cfg.update_time: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 @@ -1071,7 +1088,7 @@ class Ticker: if self.show_adj else '' ) return self.fs_str.format( - lbl = (self.create_label(id) if gcfg.name_labels else d['symbol']), + lbl = self.create_label(id) if cfg.name_labels else d['symbol'], p_spot = green(p_spot) if id in self.hl_ids else p_spot, p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, upd = d.get('last_updated_fmt'), diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 3170c6e..1876a63 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev3 +3.3.dev4 diff --git a/setup.cfg b/setup.cfg index f0c6629..e3ef90e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev19 + mmgen-wallet>=14.1.dev21 pyyaml yahooquery From 18a92cd461c88cc1b1b48b3cf8f969d20fc79088 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 19 Jul 2024 09:53:52 +0000 Subject: [PATCH 29/81] mmnode-ticker: add `--quiet` option, back up cached JSON data --- mmgen_node_tools/Ticker.py | 17 +++++++++++++---- mmgen_node_tools/main_ticker.py | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index fd05f99..bae27c2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -136,17 +136,24 @@ class DataSource: if not data: if use_cached_data: - die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host') + die(1, + f'No cached {self.data_desc}! Run command without the --cached-data option, ' + 'or use --download to retrieve data from remote host') else: die(2,'Remote host returned no data!') elif 'error' in data: die(1,data['error']) if use_cached_data: - msg(f'Using cached data from ~/{self.json_fn_rel}') + if not cfg.quiet: + msg(f'Using cached data from ~/{self.json_fn_rel}') else: - open(self.json_fn,'w').write(json_text) - msg(f'JSON data cached to ~/{self.json_fn_rel}') + if os.path.exists(self.json_fn): + os.rename(self.json_fn, self.json_fn + '.bak') + with open(self.json_fn, 'w') as fh: + fh.write(json_text) + if not cfg.quiet: + msg(f'JSON data cached to ~/{self.json_fn_rel}') if gcfg.download: sys.exit(0) @@ -679,6 +686,7 @@ def make_cfg(gcfg_arg): 'pager', 'thousands_comma', 'update_time', + 'quiet', 'verbose']) global gcfg,cfg_in,src_cls,cfg @@ -732,6 +740,7 @@ def make_cfg(gcfg_arg): pager = gcfg.pager or cfg_in.cfg.get('pager'), thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), + quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), ) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index f758089..c3f7325 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -53,6 +53,7 @@ opts_data = { columns ‘C’ (specify with comma-separated letters {pc}) -P, --pager Pipe the output to a pager +-q, --quiet Produce quieter output -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. From 0c54cb2c974943ef3a2301b3c6f5892af1e32cf8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 19 Jul 2024 09:54:03 +0000 Subject: [PATCH 30/81] Version 3.3.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1876a63..15a2799 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.dev4 +3.3.0 diff --git a/setup.cfg b/setup.cfg index e3ef90e..1945c49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.dev21 + mmgen-wallet==14.1.0 pyyaml yahooquery From 1a9184debc2f09205abd13a406619e368d7c54dc Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 20 Jul 2024 16:45:56 +0000 Subject: [PATCH 31/81] version bump --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 15a2799..49cbc5f 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.0 +3.4.dev0 diff --git a/setup.cfg b/setup.cfg index 1945c49..def5f65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet==14.1.0 + mmgen-wallet>=14.1.0 pyyaml yahooquery From c9d6d8f0475f003a7a86f7d5d005ed94495d5bbd Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 22 Sep 2024 16:10:08 +0000 Subject: [PATCH 32/81] Version 3.4.0 - released concurrently with MMGen Wallet v15.0.0 - adds macOS support --- README.md | 6 +++--- mmgen_node_tools/data/keywords | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 16 +++++++++++++--- test/cmdtest_py_d/cfg.py | 7 ++++++- test/cmdtest_py_d/ct_main.py | 2 +- test/init.sh | 13 ++++++++++--- 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f72051f..2751da7 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > $ python3 -m pip install --no-deps yahooquery > ``` -#### Linux: +#### Linux, macOS: -> Install the [required MMGen Wallet packages][7] for your Linux distribution. +> Install some [required packages][7] with your package manager and pip. ### Stable version: @@ -100,5 +100,5 @@ Donate: [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ -[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux +[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-Wallet-on-Linux-or-macOS [8]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Microsoft-Windows#a_m diff --git a/mmgen_node_tools/data/keywords b/mmgen_node_tools/data/keywords index 3c3c563..f26b502 100644 --- a/mmgen_node_tools/data/keywords +++ b/mmgen_node_tools/data/keywords @@ -1 +1 @@ -Bitcoin, BTC, Ethereum, ETH, Monero, XMR, ERC20, cryptocurrency, wallet, BIP32, cold storage, offline, online, spending, open-source, command-line, Python, Linux, Bitcoin Core, bitcoind, hd, deterministic, hierarchical, secure, anonymous, Electrum, seed, mnemonic, brainwallet, Scrypt, utility, script, scriptable, blockchain, raw, transaction, permissionless, console, terminal, curses, ansi, color, tmux, remote, client, daemon, RPC, json, entropy, xterm, rxvt, PowerShell, MSYS, MSYS2, MinGW, MinGW64, MSWin, Armbian, Raspbian, Raspberry Pi, Orange Pi, BCash, BCH, Litecoin, LTC, altcoin, ZEC, Zcash, DASH, Dashpay, SHA256Compress, monerod, EMC, Emercoin, token, deploy, contract, gas, fee, smart contract, solidity, Parity, OpenEthereum, testnet, devmode, Kovan +Bitcoin, BTC, Ethereum, ETH, Monero, XMR, ERC20, cryptocurrency, wallet, cold storage, offline, signing, online, security, privacy, spending, financial, investment, open-source, command-line, Python, Linux, Microsoft Windows, macOS, Bitcoin Core, BIP32, BIP39, BIP44, BIP69, BIP125, bitcoind, hd, deterministic, hierarchical, secure, anonymous, Electrum, seed, mnemonic, brainwallet, Scrypt, utility, script, scriptable, blockchain, raw, transaction, permissionless, console, terminal, curses, ansi, color, tmux, remote, client, daemon, RPC, json, entropy, xterm, rxvt, MSYS2, MSWin, Armbian, Raspbian, Raspberry Pi, Orange Pi, Rock Pi, BCash, Bitcoin Cash Node, BCH, Litecoin, LTC, altcoin, ZEC, Zcash, SHA256Compress, monerod, token, deploy, contract, gas, fee, smart contract, solidity, Parity, OpenEthereum, testnet, devmode, regtest diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 49cbc5f..1809198 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.4.dev0 +3.4.0 diff --git a/setup.cfg b/setup.cfg index def5f65..9a0ab04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,19 +8,29 @@ author = The MMGen Project author_email = mmgen@tuta.io url = https://github.com/mmgen/mmgen-node-tools license = GNU GPL v3 -platforms = Linux, Armbian, Raspbian, MS Windows +platforms = Linux, Armbian, Raspbian, MS Windows, MacOS keywords = file: mmgen_node_tools/data/keywords project_urls = Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = - Programming Language :: Python :: 3 License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows + Operating System :: MacOS Environment :: Console + Programming Language :: Python :: 3 + Programming Language :: C + Framework :: AsyncIO + Framework :: aiohttp Topic :: Office/Business :: Financial Topic :: Security :: Cryptography + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + Intended Audience :: Developers + Intended Audience :: End Users/Desktop + Intended Audience :: Financial and Insurance Industry + Intended Audience :: System Administrators Development Status :: 5 - Production/Stable [options] @@ -28,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.0 + mmgen-wallet==15.0.0 pyyaml yahooquery diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_py_d/cfg.py index 96bfe9f..ef52880 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -30,4 +30,9 @@ cfgs = { } def fixup_cfgs(): - pass + import os + + for k in cfgs: + cfgs[k]['tmpdir'] = os.path.join('test', 'tmp', str(k)) + +fixup_cfgs() diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index 7c38bcf..d52a2e3 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -96,4 +96,4 @@ class CmdTestMain(CmdTestBase): def peerblocks3(self): return self.peerblocks2( ['--columns=80'], - pexpect_spawn = sys.platform != 'win32' ) + pexpect_spawn = sys.platform == 'linux') diff --git a/test/init.sh b/test/init.sh index 51e4e9e..a379adf 100755 --- a/test/init.sh +++ b/test/init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet # Copyright (C)2013-2022 The MMGen Project @@ -18,6 +18,12 @@ trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT trap 'echo -e "${RED}Node Tools test suite initialization exited with error (line $BASH_LINENO) $RESET"' ERR umask 0022 +for i in '-c' '-f'; do + stat $i %i / >/dev/null 2>&1 && stat_fmt_opt=$i +done + +[ "$stat_fmt_opt" ] || { echo 'No suitable ‘stat’ binary found. Cannot proceed'; exit; } + STDOUT_DEVNULL='>/dev/null' STDERR_DEVNULL='2>/dev/null' @@ -77,6 +83,7 @@ create_test_links() { test/overlay/__init__.py symbolic test/overlay/fakemods/mmgen symbolic test/__init__.py symbolic + test/clean.py symbolic test/cmdtest.py hard test/unit_tests.py hard test/test-release.sh symbolic @@ -87,7 +94,7 @@ create_test_links() { while read path type; do [ "$path" ] || continue pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') - symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) + symlink_arg=$(if [ $type == 'symbolic' ]; then echo -s; fi) target="$wallet_repo/$path" if [ ! -e "$target" ]; then echo "Target path $target is missing! Cannot proceed" @@ -99,7 +106,7 @@ create_test_links() { [ "$VERBOSE" ] && printf "$fs" "Deleting" "symbolic link:" $path $target rm -rf $path elif [ -e $path ]; then - if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then + if [ "$(stat $stat_fmt_opt %i $path)" -ne "$(stat $stat_fmt_opt %i $target)" ]; then [ "$VERBOSE" ] && printf "$fs" "Deleting" "stale hard link:" $path "?" rm -rf $path fi From 92fdfc047e213aed42fdb7a71c29aef32707404f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 29 Sep 2024 14:29:10 +0000 Subject: [PATCH 33/81] Support BCH cashaddr format --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- setup.cfg | 2 +- test/cmdtest_py_d/ct_regtest.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1809198..3f8a888 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.4.0 +3.5.dev1 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7af2b77..e78a53e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -38,7 +38,7 @@ def do_output(proto,addr_data,blk_hdrs): indent = ' ' * (col1w + 2) for n,(addr,unspents) in enumerate(addr_data.items(),1): - Msg(f'\n{n:{col1w}}) Address: {addr.hl()}') + Msg(f'\n{n:{col1w}}) Address: {addr.hl(addr.view_pref)}') if unspents: heights = { u['height'] for u in unspents } @@ -93,7 +93,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): if unspents: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(width=max_addrw,color=True), + a = addr.fmt(addr.view_pref, width=max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), @@ -104,7 +104,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): else: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(width=max_addrw,color=True), + a = addr.fmt(addr.view_pref, width=max_addrw, color=True), u = ' -', b = '-', t = '', diff --git a/setup.cfg b/setup.cfg index 9a0ab04..875bb01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet==15.0.0 + mmgen-wallet>=15.1.dev1 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index c89f8fa..ee59b31 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -112,7 +112,7 @@ class CmdTestRegtest(CmdTestBase): if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) - self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) + self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto,'regtest',[1,2,3,4,5])] self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) From cb42eaa8cdc602be0d3718d73331b94745558d36 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 8 Oct 2024 13:19:15 +0000 Subject: [PATCH 34/81] update for MMGen Wallet 15.1.dev3 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 2 +- test/cmdtest_py_d/ct_regtest.py | 93 +++++++++++++++++---------------- 5 files changed, 53 insertions(+), 48 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 3f8a888..ecc3a64 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev1 +3.5.dev2 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c3f7325..351cb1e 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -234,7 +234,7 @@ from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config from . import Ticker -gcfg = Config( opts_data=opts_data, do_post_init=True ) +gcfg = Config(opts_data=opts_data, caller_post_init=True) Ticker.make_cfg(gcfg) diff --git a/setup.cfg b/setup.cfg index 875bb01..9b82f69 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev1 + mmgen-wallet>=15.1.dev3 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 8127e1d..6ac7d37 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -50,7 +50,7 @@ class CmdTestHelp(CmdTestBase): return t def longhelpscreens(self): - return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') + return self.helpscreens(arg='--longhelp',expect='USAGE:.*GLOBAL OPTIONS:') class CmdTestScripts(CmdTestBase): 'scripts not requiring a coin daemon' diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index ee59b31..0807a16 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -35,7 +35,6 @@ class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' networks = ('btc','ltc','bch') passthru_opts = ('coin',) - extra_spawn_args = ['--regtest=1'] tmpdir_nums = [1] color = True deterministic = False @@ -177,57 +176,57 @@ class CmdTestRegtest(CmdTestBase): def sendto2(self): return self.sendto(self.addrs[0],'0.234') def sendto3(self): return self.sendto(self.addrs[1],'0.345') - def addrbal(self,args,expect_list): - t = self.spawn('mmnode-addrbal',args) + def addrbal(self, args, expect_list): + t = self.spawn('mmnode-addrbal', args2 + args) t.match_expect_list(expect_list) return t def addrbal_single(self): return self.addrbal( - args2 + [self.addrs[0]], + [self.addrs[0]], [ f'Balance: 0.357 {cfg.coin}', '2 unspent outputs in 2 blocks', - '394','0.123', - '395','0.234' + '394', '0.123', + '395', '0.234' ]) def addrbal_multiple(self): return self.addrbal( - args2 + [self.addrs[1],self.addrs[0]], + [self.addrs[1], self.addrs[0]], [ - '396','0.345', - '394','0.123', - '395','0.234' + '396', '0.345', + '394', '0.123', + '395', '0.234' ]) def addrbal_multiple_tabular1(self): return self.addrbal( - args2 + ['--tabular',self.addrs[1],self.addrs[0]], + ['--tabular', self.addrs[1], self.addrs[0]], [ - self.addrs[1] + ' 1 396','0.345', - self.addrs[0] + ' 2 395','0.357' + self.addrs[1] + ' 1 396', '0.345', + self.addrs[0] + ' 2 395', '0.357' ]) def addrbal_multiple_tabular2(self): return self.addrbal( - args2 + ['--tabular','--first-block',self.addrs[1],self.addrs[0]], + ['--tabular', '--first-block', self.addrs[1], self.addrs[0]], [ - self.addrs[1] + ' 1 396','396','0.345', - self.addrs[0] + ' 2 394','395','0.357' + self.addrs[1] + ' 1 396', '396', '0.345', + self.addrs[0] + ' 2 394', '395', '0.357' ]) def addrbal_nobal1(self): return self.addrbal( - args2 + [self.addrs[2]], ['Address has no balance'] ) + [self.addrs[2]], ['Address has no balance']) def addrbal_nobal2(self): return self.addrbal( - args2 + [self.addrs[2],self.addrs[3]], ['Addresses have no balances'] ) + [self.addrs[2], self.addrs[3]], ['Addresses have no balances']) def addrbal_nobal3(self): return self.addrbal( - args2 + [self.addrs[4],self.addrs[0],self.addrs[3]], + [self.addrs[4], self.addrs[0], self.addrs[3]], [ 'No balance', '2 unspent outputs in 2 blocks', @@ -237,7 +236,7 @@ class CmdTestRegtest(CmdTestBase): def addrbal_nobal3_tabular1(self): return self.addrbal( - args2 + ['--tabular',self.addrs[4],self.addrs[0],self.addrs[3]], + ['--tabular', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - -', self.addrs[0] + ' 2 395','0.357', @@ -246,7 +245,7 @@ class CmdTestRegtest(CmdTestBase): def addrbal_nobal3_tabular2(self): return self.addrbal( - args2 + ['--tabular','--first-block',self.addrs[4],self.addrs[0],self.addrs[3]], + ['--tabular', '--first-block', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - - -', self.addrs[0] + ' 2 394','395','0.357', @@ -254,39 +253,45 @@ class CmdTestRegtest(CmdTestBase): ]) def blocks_info(self,args,expect_list): - t = self.spawn('mmnode-blocks-info',args) + t = self.spawn('mmnode-blocks-info', args1 + args) t.match_expect_list(expect_list) return t def blocks_info1(self): - return self.blocks_info( args1 + ['--help'], ['USAGE:','OPTIONS:']) + return self.blocks_info( + ['--help'], + ['USAGE:','OPTIONS:']) def blocks_info2(self): - return self.blocks_info( args1, [ - 'Current height: 396', - ]) + return self.blocks_info( + [], + ['Current height: 396']) def blocks_info3(self): - return self.blocks_info( args1 + ['+100'], [ - 'Range: 297-396', - 'Current height: 396', - 'Next diff adjust: 2016' - ]) + return self.blocks_info( + ['+100'], + [ + 'Range: 297-396', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) def blocks_info4(self): n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) - return self.blocks_info( args1 + ['--miner-info','--fields=all','--stats=all','+3'], [ - 'Averages', - f'nTx: {n1}', - f'Inputs: {i1}', - f'Outputs: {o1}', - 'Totals', - f'nTx: {n2}', - f'Inputs: {i2}', - f'Outputs: {o2}', - 'Current height: 396', - 'Next diff adjust: 2016' - ]) + return self.blocks_info( + ['--miner-info', '--fields=all', '--stats=all', '+3'], + [ + 'Averages', + f'nTx: {n1}', + f'Inputs: {i1}', + f'Outputs: {o1}', + 'Totals', + f'nTx: {n2}', + f'Inputs: {i2}', + f'Outputs: {o2}', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) async def feeview_setup(self): @@ -380,7 +385,7 @@ class CmdTestRegtest(CmdTestBase): return 'ok' def _feeview(self,args,expect_list=[]): - t = self.spawn('mmnode-feeview',args) + t = self.spawn('mmnode-feeview', args1 + args) if expect_list: t.match_expect_list(expect_list) return t From dee57d88869a8c9c6c22bbf01bc07de35de7545e Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:47 +0000 Subject: [PATCH 35/81] update for MMGen Wallet CoinAmt changes --- mmgen_node_tools/BlocksInfo.py | 27 +++++++++++++++++++-------- mmgen_node_tools/main_addrbal.py | 4 ++-- test/cmdtest_py_d/ct_regtest.py | 9 +++++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 40cc6e5..26e6cfa 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -23,6 +23,7 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of block import re,json from collections import namedtuple from time import strftime,gmtime +from decimal import Decimal from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int from mmgen.rpc import json_encoder @@ -241,14 +242,14 @@ class BlocksInfo: 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), 'fe': lambda arg: str(arg), - 'di': lambda arg: '{:.2e}'.format(arg), + 'di': lambda arg: '{:.2e}'.format(Decimal(arg)), } if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), - 'fe': lambda arg: str(int(arg * to_satoshi)), - 'tf': lambda arg: '{:.8f}'.format(arg), + 'fe': lambda arg: str(int(Decimal(arg) * to_satoshi)), + 'tf': lambda arg: '{:.8f}'.format(Decimal(arg)), }) self.fnames = tuple( @@ -547,7 +548,7 @@ class BlocksInfo: else: sample_blks = min(min_sample_blks,self.tip) start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) - diff_adj = float(tip_hdr['difficulty'] / start_hdr['difficulty']) + diff_adj = Decimal(tip_hdr['difficulty']) / Decimal(start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] bdi = ((time1 * diff_adj) + time2) / sample_blks @@ -570,17 +571,26 @@ class BlocksInfo: 'sample_blks': ('{}', sample_blks) } ), - ('Cur difficulty: {}', 'cur_diff', '{:.2e}', tip_hdr['difficulty']), + ('Cur difficulty: {}', 'cur_diff', '{:.2e}', Decimal(tip_hdr['difficulty'])), ('Est. diff adjust: {}%', 'est_diff_adjust_pct', '{:+.2f}', ((600 / bdi) - 1) * 100), )) + def sum_field_avg(self, field): + return self.sum_field_total(field) // len(self.res) + + def sum_field_total(self, field): + if isinstance(getattr(self.res[0], field), str): + return sum(Decimal(getattr(block, field)) for block in self.res) + else: + return sum(getattr(block, field) for block in self.res) + async def create_col_avg_stats(self): def gen(): for field in self.fnames: if field in self.avg_stats_skip: yield ( field, ('{}','') ) else: - ret = sum(getattr(block,field) for block in self.res) // len(self.res) + ret = self.sum_field_avg(field) func = self.fields[field].fmt_func yield ( field, ( (self.fmt_funcs[func] if func else '{}'), ret )) if not self.header_printed: @@ -590,9 +600,10 @@ class BlocksInfo: def avg_stats_data(self,data,spec_conv,spec_val): coin = self.rpc.proto.coin + return data( hdr = 'Averages for processed blocks:', - func = lambda field: sum(getattr(block,field) for block in self.res) // len(self.res), + func = self.sum_field_avg, spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}' }, spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), @@ -616,7 +627,7 @@ class BlocksInfo: coin = self.rpc.proto.coin return data( hdr = 'Totals for processed blocks:', - func = lambda field: sum(getattr(block,field) for block in self.res), + func = self.sum_field_total, spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}' }, spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index e78a53e..7dd37a3 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -44,7 +44,7 @@ def do_output(proto,addr_data,blk_hdrs): heights = { u['height'] for u in unspents } Msg('{}Balance: {}'.format( indent, - proto.coin_amt(sum(u['amount'] for u in unspents)).hl2(unit=True,fs='{:,}') )), + sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), Msg('{}{} unspent output{} in {} block{}'.format( indent, red(str(len(unspents))), @@ -99,7 +99,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = proto.coin_amt(sum(u['amount'] for u in unspents)).fmt(color=True,iwidth=7,prec=8) + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(color=True, iwidth=7, prec=8) )) else: Msg(fs.format( diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index 0807a16..f1b2774 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -13,6 +13,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import sys,os +from decimal import Decimal from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto @@ -316,13 +317,13 @@ class CmdTestRegtest(CmdTestBase): # very approximate tx size estimation: ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) - x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt(self.proto.coin_amt.satoshi) + x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt.satoshi n = n_in - 1 vmax = high - low for i in range(n_in): - yield (low + (i/n)**6 * vmax) * x + yield Decimal(low + (i/n)**6 * vmax) * x async def do_tx(inputs,outputs,wif): tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) @@ -335,14 +336,14 @@ class CmdTestRegtest(CmdTestBase): tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} - outputs.update({burn_addr: tx_input['amount'] - (tx1_amt*nTxs) - fee}) + outputs.update({burn_addr: self.proto.coin_amt(tx_input['amount']) - (tx1_amt*nTxs) - fee}) return await do_tx( [{ 'txid': tx_input['txid'], 'vout': 0 }], outputs, await r.miner_wif) async def do_tx2(tx,pairno): - fee = fees[pairno] + fee = self.proto.coin_amt(fees[pairno], from_decimal=True) outputs = {p.addr:tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) return await do_tx( From b9b289c56db227cfc42e541dffc7767bce129c78 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:48 +0000 Subject: [PATCH 36/81] directory rename: - test/cmdtest_py_d -> test/cmdtest_d --- MANIFEST.in | 2 +- test/{cmdtest_py_d => cmdtest_d}/cfg.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_main.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_misc.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_regtest.py | 2 +- test/init.sh | 4 ++-- test/test-release.d/cfg.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename test/{cmdtest_py_d => cmdtest_d}/cfg.py (93%) rename test/{cmdtest_py_d => cmdtest_d}/ct_main.py (96%) rename test/{cmdtest_py_d => cmdtest_d}/ct_misc.py (99%) rename test/{cmdtest_py_d => cmdtest_d}/ct_regtest.py (99%) diff --git a/MANIFEST.in b/MANIFEST.in index 370da07..1fe78fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,6 @@ include mmgen_node_tools/data/* include test/init.sh include test/test-release.d/*.sh include test/unit_tests_d/*.py -include test/cmdtest_py_d/*.py +include test/cmdtest_d/*.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_d/cfg.py similarity index 93% rename from test/cmdtest_py_d/cfg.py rename to test/cmdtest_d/cfg.py index ef52880..c28ce5d 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_d/cfg.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.cfg: configuration data for cmdtest.py +test.cmdtest_d.cfg: configuration data for cmdtest.py """ cmd_groups_altcoin = [] diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_d/ct_main.py similarity index 96% rename from test/cmdtest_py_d/ct_main.py rename to test/cmdtest_d/ct_main.py index d52a2e3..3009d2f 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_d/ct_main.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-wallet """ -cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite +cmdtest_d.ct_main: Basic operations tests for the cmdtest.py test suite """ import sys,time diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_d/ct_misc.py similarity index 99% rename from test/cmdtest_py_d/ct_misc.py rename to test/cmdtest_d/ct_misc.py index 6ac7d37..15b109e 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_d/ct_misc.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite +test.cmdtest_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite """ import os,shutil diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_d/ct_regtest.py similarity index 99% rename from test/cmdtest_py_d/ct_regtest.py rename to test/cmdtest_d/ct_regtest.py index f1b2774..24f19b9 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_d/ct_regtest.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite +test.cmdtest_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import sys,os diff --git a/test/init.sh b/test/init.sh index a379adf..8f77c82 100755 --- a/test/init.sh +++ b/test/init.sh @@ -87,8 +87,8 @@ create_test_links() { test/cmdtest.py hard test/unit_tests.py hard test/test-release.sh symbolic - test/cmdtest_py_d/common.py symbolic - test/cmdtest_py_d/ct_base.py symbolic + test/cmdtest_d/common.py symbolic + test/cmdtest_d/ct_base.py symbolic cmds/mmgen-regtest symbolic ' while read path type; do diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index b371e72..7a393df 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -42,7 +42,7 @@ init_tests() { t_lint=" - $pylint --errors-only mmgen_node_tools - $pylint --errors-only test - - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_py_d + - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_d " d_unit="low-level subsystems" From 30772b369999c6b5deae7d4ea653d46cc09680ed Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:48 +0000 Subject: [PATCH 37/81] Update for MMGen Wallet 15.1.dev6 --- MANIFEST.in | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/init.sh | 8 +++++++- test/{unit_tests_d => modtest_d}/ut_BlocksInfo.py | 0 test/{unit_tests_d => modtest_d}/ut_dep.py | 0 test/test-release.d/cfg.sh | 12 ++++++------ 7 files changed, 16 insertions(+), 10 deletions(-) rename test/{unit_tests_d => modtest_d}/ut_BlocksInfo.py (100%) rename test/{unit_tests_d => modtest_d}/ut_dep.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 1fe78fa..0443a99 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include mmgen_node_tools/data/* include test/init.sh include test/test-release.d/*.sh -include test/unit_tests_d/*.py +include test/modtest_d/*.py include test/cmdtest_d/*.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index ecc3a64..f19fa82 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev2 +3.5.dev3 diff --git a/setup.cfg b/setup.cfg index 9b82f69..edbad57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev3 + mmgen-wallet>=15.1.dev6 pyyaml yahooquery diff --git a/test/init.sh b/test/init.sh index 8f77c82..f6a6f00 100755 --- a/test/init.sh +++ b/test/init.sh @@ -77,6 +77,10 @@ create_dir_links() { done } +delete_old_stuff() { + rm -rf test/unit_tests.py +} + create_test_links() { paths=' test/include symbolic @@ -85,7 +89,7 @@ create_test_links() { test/__init__.py symbolic test/clean.py symbolic test/cmdtest.py hard - test/unit_tests.py hard + test/modtest.py hard test/test-release.sh symbolic test/cmdtest_d/common.py symbolic test/cmdtest_d/ct_base.py symbolic @@ -121,6 +125,8 @@ create_test_links() { becho 'Initializing MMGen Node Tools Test Suite' +delete_old_stuff + check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/modtest_d/ut_BlocksInfo.py similarity index 100% rename from test/unit_tests_d/ut_BlocksInfo.py rename to test/modtest_d/ut_BlocksInfo.py diff --git a/test/unit_tests_d/ut_dep.py b/test/modtest_d/ut_dep.py similarity index 100% rename from test/unit_tests_d/ut_dep.py rename to test/modtest_d/ut_dep.py diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 7a393df..a34c324 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -18,7 +18,7 @@ # mmnode-ticker OK # mmnode-txfind - -all_tests='unit lint misc scripts btc btc_rt bch_rt ltc_rt' +all_tests='mod lint misc scripts btc btc_rt bch_rt ltc_rt' groups_desc=" default - All tests minus the extra tests @@ -29,10 +29,10 @@ groups_desc=" " init_groups() { - dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + dfl_tests='mod misc scripts btc btc_rt bch_rt ltc_rt' extra_tests='lint' - noalt_tests='unit misc scripts btc btc_rt' - quick_tests='unit misc scripts btc btc_rt' + noalt_tests='mod misc scripts btc btc_rt' + quick_tests='mod misc scripts btc btc_rt' qskip_tests='lint bch_rt ltc_rt' } @@ -45,8 +45,8 @@ init_tests() { - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_d " - d_unit="low-level subsystems" - t_unit="- $unit_tests_py" + d_mod="low-level subsystems" + t_mod="- $modtest_py" d_misc="miscellaneous features" t_misc="- $cmdtest_py helpscreens" From bec7df1d6b5b3ea342fc28292ceaf83dc7498054 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 3 Jan 2025 16:42:11 +0300 Subject: [PATCH 38/81] support Nix and NixOS Quick Start for BTC: $ git clone https://github.com/mmgen/mmgen-wallet $ git clone https://github.com/mmgen/mmgen-node-tools $ cd mmgen-node-tools $ test/init.sh $ nix-shell --pure nix Enable altcoins and additional packages: # From the mmgen-node-tools repository root: $ mkdir -p ~/.mmgen $ cp ../mmgen-wallet/nix/user-packages.nix ~/.mmgen # ... edit ~/.mmgen/user-packages.nix as required ... $ nix-shell --pure nix For NixOS installation and other information, see: nix/README.node-tools ../mmgen-wallet/nix/README --- mmgen_node_tools/data/version | 2 +- nix/README.node-tools | 11 ++++++++ nix/default.nix | 6 +++++ nix/node-tools-packages.nix | 12 +++++++++ nix/shell.nix | 1 + nix/yahooquery-noversioning.patch | 45 +++++++++++++++++++++++++++++++ nix/yahooquery.nix | 44 ++++++++++++++++++++++++++++++ setup.cfg | 2 +- 8 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 nix/README.node-tools create mode 100644 nix/default.nix create mode 100644 nix/node-tools-packages.nix create mode 100644 nix/shell.nix create mode 100644 nix/yahooquery-noversioning.patch create mode 100644 nix/yahooquery.nix diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f19fa82..cf8acd2 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev3 +3.5.dev4 diff --git a/nix/README.node-tools b/nix/README.node-tools new file mode 100644 index 0000000..9308181 --- /dev/null +++ b/nix/README.node-tools @@ -0,0 +1,11 @@ +Nix configuration directory for the MMGen Node Tools suite + +Usage is as described in ‘nix/README’ in the mmgen-wallet repository, with the +following differences: + + a) all commands are executed from the repository root of mmgen-node-tools + instead of mmgen-wallet + + b) for NixOS, complete the steps as described up until the rebuild step. + Copy the contents of this directory to ‘/etc/nixos/mmgen-project’ (this + will overwrite ‘default.nix’), and continue with the rebuild step. diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..7b4b214 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,6 @@ +import ( + if builtins.pathExists ./merged-packages.nix then + ./merged-packages.nix + else + ../../mmgen-wallet/nix/merged-packages.nix + ) { add_pkgs_path = ./node-tools-packages.nix; } diff --git a/nix/node-tools-packages.nix b/nix/node-tools-packages.nix new file mode 100644 index 0000000..19792c7 --- /dev/null +++ b/nix/node-tools-packages.nix @@ -0,0 +1,12 @@ +{ pkgs, python }: + +{ + system-packages = with pkgs; { + cacert = cacert; # ticker (curl) + }; + + python-packages = with python.pkgs; { + yahooquery = (pkgs.callPackage ./yahooquery.nix {}); # ticker + pyyaml = pyyaml; # ticker + }; +} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 0000000..01991d2 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1 @@ +import ../../mmgen-wallet/nix/shell.nix { add_pkgs_path = ./node-tools-packages.nix; } diff --git a/nix/yahooquery-noversioning.patch b/nix/yahooquery-noversioning.patch new file mode 100644 index 0000000..4bd3297 --- /dev/null +++ b/nix/yahooquery-noversioning.patch @@ -0,0 +1,45 @@ +diff --git a/pyproject.toml b/pyproject.toml +index 9d3fb29..399c215 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -10,24 +10,24 @@ readme = "README.md" + + [tool.poetry.dependencies] + python = ">=3.8.1,<4.0" +-requests = "^2.31.0" +-pandas = "^2.0.3" +-requests-futures = "^1.0.1" +-tqdm = "^4.65.0" +-lxml = "^4.9.3" +-selenium = {version = "^4.10.0", optional = true} +-beautifulsoup4 = "^4.12.2" ++requests = ">=2.31.0" ++pandas = ">=2.0.3" ++requests-futures = ">=1.0.1" ++tqdm = ">=4.65.0" ++lxml = ">=4.9.3" ++selenium = {version = ">=4.10.0", optional = true} ++beautifulsoup4 = ">=4.12.2" + + [tool.poetry.dev-dependencies] +-pytest = "^7.4.0" +-isort = "^5.0.0" +-flake8 = "^6.0.0" +-mypy = "^1.4.1" +-pytest-cov = "^4.1.0" +-black = "^23.7.0" +-pre-commit = "^3.3.3" +-ipython = "^8.0.0" +-mkdocs-material = "^9.1.18" ++pytest = ">=7.4.0" ++isort = ">=5.0.0" ++flake8 = ">=6.0.0" ++mypy = ">=1.4.1" ++pytest-cov = ">=4.1.0" ++black = ">=23.7.0" ++pre-commit = ">=3.3.3" ++ipython = ">=8.0.0" ++mkdocs-material = ">=9.1.18" + + [build-system] + requires = ["poetry-core>=1.0.0"] diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix new file mode 100644 index 0000000..8b1535d --- /dev/null +++ b/nix/yahooquery.nix @@ -0,0 +1,44 @@ +{ + lib, + pkgs, + fetchFromGitHub, +}: + +with pkgs.python312.pkgs; + +buildPythonPackage rec { + pname = "yahooquery"; + version = "2.3.7"; + pyproject = true; + + disabled = pythonOlder "3.8.1"; + + src = fetchFromGitHub { + owner = "dpguthrie"; + repo = "yahooquery"; + rev = "refs/tags/v${version}"; + hash = "sha256-Iyuni1SoTB6f7nNFhN5A8Gnv9kV78frjpqvvW8qd+/M="; + }; + + patches = [ ./yahooquery-noversioning.patch ]; + + build-system = [ poetry-core ]; + + dependencies = [ + requests # ^2.31.0 + pandas # ^2.0.3 + requests-futures # ^1.0.1 + tqdm # ^4.65.0 + lxml # ^4.9.3 + selenium # {version = ^4.10.0, optional = true} + beautifulsoup4 # ^4.12.2 + ]; + + doCheck = false; # skip tests + + meta = with lib; { + description = "Python wrapper for an unofficial Yahoo Finance API"; + homepage = "https://yahooquery.dpguthrie.com"; + license = licenses.mit; + }; +} diff --git a/setup.cfg b/setup.cfg index edbad57..f63dd91 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev6 + mmgen-wallet>=15.1.dev9 pyyaml yahooquery From 7cb2fc5b08722b3a2ac1436fb9ce2d7b4fd12852 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Jan 2025 11:06:28 +0000 Subject: [PATCH 39/81] nix/shell.nix: add `repo` arg; test/init.sh: add `-c` option --- mmgen_node_tools/data/version | 2 +- nix/shell.nix | 5 ++++- setup.cfg | 2 +- test/init.sh | 17 ++++++++++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index cf8acd2..c087ce9 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev4 +3.5.dev5 diff --git a/nix/shell.nix b/nix/shell.nix index 01991d2..acfdee3 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1 +1,4 @@ -import ../../mmgen-wallet/nix/shell.nix { add_pkgs_path = ./node-tools-packages.nix; } +import ../../mmgen-wallet/nix/shell.nix { + repo = "mmgen-node-tools"; + add_pkgs_path = ./node-tools-packages.nix; +} diff --git a/setup.cfg b/setup.cfg index f63dd91..db836e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev9 + mmgen-wallet>=15.1.dev10 pyyaml yahooquery diff --git a/test/init.sh b/test/init.sh index f6a6f00..15eeddb 100755 --- a/test/init.sh +++ b/test/init.sh @@ -28,15 +28,17 @@ STDOUT_DEVNULL='>/dev/null' STDERR_DEVNULL='2>/dev/null' PROGNAME=$(basename $0) -while getopts hv OPT +while getopts hcv OPT do case "$OPT" in h) printf " %-16s Initialize the MMGen Node Tools test suite\n" "${PROGNAME}:" echo " USAGE: $PROGNAME" echo " OPTIONS: '-h' Print this help message" + echo " -c Create links from mmgen-wallet ‘cmds’ subdirectory" echo " -v Be more verbose" exit ;; v) VERBOSE=1 STDOUT_DEVNULL='' STDERR_DEVNULL='' ;; + c) CMD_LINKS=1 ;; *) exit ;; esac done @@ -123,6 +125,17 @@ create_test_links() { done <<<$paths } +create_cmd_links() { + [ "$VERBOSE" ] && becho 'Creating links to mmgen-wallet repo ‘cmds’ subdirectory' + ( + filenames=$(cd $wallet_repo/cmds && ls) + cd cmds + for filename in $filenames; do + [ -e $filename ] || ln -s "../$wallet_repo/cmds/$filename" + done + ) +} + becho 'Initializing MMGen Node Tools Test Suite' delete_old_stuff @@ -137,6 +150,8 @@ create_dir_links create_test_links +[ "$CMD_LINKS" ] && create_cmd_links + [ "$VERBOSE" ] && becho 'OK' true From b62ff991fe030448c52440f5ccf722b070ed2d41 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 6 Feb 2025 10:07:57 +0000 Subject: [PATCH 40/81] update for MMGen Wallet v15.1.dev14 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- setup.cfg | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index c087ce9..426110c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev5 +3.5.dev6 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7dd37a3..7fafac1 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -142,10 +142,10 @@ async def main(req_addrs): addr = re.match('addr\((.*?)\)',unspent['desc'])[1] addr_data[addr].append(unspent) else: - from mmgen.proto.btc.tx.base import scriptPubKey2addr + from mmgen.proto.btc.tx.base import decodeScriptPubKey for unspent in sorted(res['unspents'],key=lambda x: x['height']): - addr = scriptPubKey2addr( proto, unspent['scriptPubKey'] )[0] - addr_data[addr].append(unspent) + ds = decodeScriptPubKey(proto, unspent['scriptPubKey']) + addr_data[ds.addr].append(unspent) good_addrs = len([v for v in addr_data.values() if v]) diff --git a/setup.cfg b/setup.cfg index db836e8..e140080 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev10 + mmgen-wallet>=15.1.dev14 pyyaml yahooquery From 966c1716505f63829bb70c063560e0d3b1cff67f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 15 Mar 2025 18:23:16 +0000 Subject: [PATCH 41/81] update for MMGen Wallet v15.1.dev20 --- README.md | 2 +- mmgen_node_tools/Ticker.py | 14 ++++++++------ mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 8 ++++---- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 8 ++++---- setup.cfg | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2751da7..89c43b2 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Code repository: Code repository mirrors: [Github](https://github.com/mmgen/mmgen-node-tools) | [Gitlab](https://gitlab.com/mmgen/mmgen-node-tools) | -[Gitflic](https://gitflic.ru/project/mmgen/mmgen-node-tools) +[Codeberg](https://codeberg.org/mmgen/mmgen-node-tools) [Keybase](https://keybase.io/mmgen) | [Twitter](https://twitter.com/TheMMGenProject) | [Reddit](https://www.reddit.com/user/mmgen-py) | diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bae27c2..252ba6c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -799,7 +799,7 @@ class Ticker: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) + fmt_func = lambda t,now: time.strftime('%F %X', time.gmtime(t)) d = self.data max_w = 0 @@ -820,15 +820,17 @@ class Ticker: msg(str(e)) pass else: - t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now ) - max_w = max(len(t_fmt),max_w) + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( + (min(t,min_t) if min_t else t), + now = now) + max_w = max(len(t_fmt), max_w) self.upd_w = max_w def init_prec(self): - exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted() ) for a in self.usr_col_assets] - self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp } - self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp } + exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted()) for a in self.usr_col_assets] + self.uprec = {k: max(0, v+4) + cfg.add_prec for k, v in exp} + self.uwid = {k: 12 + max(0, abs(v)-6) + cfg.add_prec for k, v in exp} def get_id(self,asset): if asset.id: diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 426110c..aa6a660 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev6 +3.5.dev7 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7fafac1..ecf2b1a 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -60,7 +60,7 @@ def do_output(proto,addr_data,blk_hdrs): make_timestr( blk_hdrs[u['height']]['time'] ), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(color=True,iwidth=6,prec=8) + proto.coin_amt(u['amount']).fmt(6, color=True, prec=8) )) else: Msg(f'{indent}No balance') @@ -93,18 +93,18 @@ def do_output_tabular(proto,addr_data,blk_hdrs): if unspents: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(addr.view_pref, width=max_addrw, color=True), + a = addr.fmt(addr.view_pref, max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(color=True, iwidth=7, prec=8) + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8) )) else: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(addr.view_pref, width=max_addrw, color=True), + a = addr.fmt(addr.view_pref, max_addrw, color=True), u = ' -', b = '-', t = '', diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 7bee773..16591b8 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -87,7 +87,7 @@ if cfg.ignore_below: ignore_below = parse_bytespec(cfg.ignore_below) precision = ( - check_int_between(cfg.precision,min_prec,max_prec,'--precision arg') + check_int_between(cfg.precision, min_prec, max_prec, desc='--precision arg') if cfg.precision else dfl_prec ) from mmgen.term import get_terminal_size diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 1648d76..caeb178 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -171,10 +171,10 @@ async def main(): c = date(t), d = ' P' if n > nhist else '' if n < nhist else ' E', e = f'{bdr:8.5f}', - f = proto.coin_amt(sub,from_unit='satoshi').fmt(iwidth=2,prec=8), - g = proto.coin_amt(mined,from_unit='satoshi').fmt(iwidth=8,prec=8), - h = proto.coin_amt(total_mined,from_unit='satoshi').fmt(iwidth=8,prec=8) - ) for n,sub,blk,mined,total_mined,bdr,t in gen_data()) + f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), + g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), + h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) + ) for n, sub, blk, mined, total_mined, bdr, t in gen_data()) ) if cfg.list: diff --git a/setup.cfg b/setup.cfg index e140080..b43abad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev14 + mmgen-wallet>=15.1.dev20 pyyaml yahooquery From d63cb9817a249618931f526e9d5029bfe45adb0d Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 23 Mar 2025 10:12:56 +0000 Subject: [PATCH 42/81] update for MMGen Wallet v15.1.dev23 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/ct_regtest.py | 4 ++-- test/init.sh | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index aa6a660..1e3f513 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev7 +3.5.dev8 diff --git a/setup.cfg b/setup.cfg index b43abad..c67cb47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev20 + mmgen-wallet>=15.1.dev23 pyyaml yahooquery diff --git a/test/cmdtest_d/ct_regtest.py b/test/cmdtest_d/ct_regtest.py index 24f19b9..f043641 100755 --- a/test/cmdtest_d/ct_regtest.py +++ b/test/cmdtest_d/ct_regtest.py @@ -105,8 +105,8 @@ class CmdTestRegtest(CmdTestBase): ), } - def __init__(self,trunner,cfgs,spawn): - CmdTestBase.__init__(self,trunner,cfgs,spawn) + def __init__(self, cfg, trunner, cfgs, spawn): + CmdTestBase.__init__(self, cfg, trunner, cfgs, spawn) if trunner == None: return if cfg._proto.testnet: diff --git a/test/init.sh b/test/init.sh index 15eeddb..94671f9 100755 --- a/test/init.sh +++ b/test/init.sh @@ -95,6 +95,8 @@ create_test_links() { test/test-release.sh symbolic test/cmdtest_d/common.py symbolic test/cmdtest_d/ct_base.py symbolic + test/cmdtest_d/runner.py symbolic + test/cmdtest_d/group_mgr.py symbolic cmds/mmgen-regtest symbolic ' while read path type; do From 4688ee94c0dfa1a73165e72a520182e724e47c47 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 29 Mar 2025 12:12:35 +0000 Subject: [PATCH 43/81] update test suite for MMGen Wallet v15.1.dev25 --- README.md | 3 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/{ => include}/cfg.py | 2 +- test/cmdtest_d/{ct_main.py => main.py} | 6 ++-- test/cmdtest_d/{ct_misc.py => misc.py} | 6 ++-- test/cmdtest_d/{ct_regtest.py => regtest.py} | 10 +++---- test/init.sh | 31 ++++++++++++-------- 8 files changed, 34 insertions(+), 28 deletions(-) rename test/cmdtest_d/{ => include}/cfg.py (93%) rename test/cmdtest_d/{ct_main.py => main.py} (94%) rename test/cmdtest_d/{ct_misc.py => misc.py} (98%) rename test/cmdtest_d/{ct_regtest.py => regtest.py} (98%) diff --git a/README.md b/README.md index 89c43b2..a536cce 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: -Install the latest development version of [MMGen Wallet][6] for your platform: +First install the latest development version of [MMGen Wallet][6] for your +platform. Then perform the following steps: ```bash $ git clone https://github.com/mmgen/mmgen-node-tools diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1e3f513..0602083 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev8 +3.5.dev9 diff --git a/setup.cfg b/setup.cfg index c67cb47..905e659 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev23 + mmgen-wallet>=15.1.dev25 pyyaml yahooquery diff --git a/test/cmdtest_d/cfg.py b/test/cmdtest_d/include/cfg.py similarity index 93% rename from test/cmdtest_d/cfg.py rename to test/cmdtest_d/include/cfg.py index c28ce5d..5a4263d 100755 --- a/test/cmdtest_d/cfg.py +++ b/test/cmdtest_d/include/cfg.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.cfg: configuration data for cmdtest.py +test.cmdtest_d.include.cfg: configuration data for cmdtest.py """ cmd_groups_altcoin = [] diff --git a/test/cmdtest_d/ct_main.py b/test/cmdtest_d/main.py similarity index 94% rename from test/cmdtest_d/ct_main.py rename to test/cmdtest_d/main.py index 3009d2f..6e35b7d 100755 --- a/test/cmdtest_d/ct_main.py +++ b/test/cmdtest_d/main.py @@ -9,13 +9,13 @@ # https://gitlab.com/mmgen/mmgen-wallet """ -cmdtest_d.ct_main: Basic operations tests for the cmdtest.py test suite +cmdtest_d.main: Basic operations tests for the cmdtest.py test suite """ -import sys,time +import sys, time from ..include.common import cfg -from .ct_base import CmdTestBase +from .base import CmdTestBase class CmdTestMain(CmdTestBase): 'basic operations with fake RPC data' diff --git a/test/cmdtest_d/ct_misc.py b/test/cmdtest_d/misc.py similarity index 98% rename from test/cmdtest_d/ct_misc.py rename to test/cmdtest_d/misc.py index 15b109e..1a8175e 100755 --- a/test/cmdtest_d/ct_misc.py +++ b/test/cmdtest_d/misc.py @@ -9,13 +9,13 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite +test.cmdtest_d.misc: Miscellaneous test groups for the cmdtest.py test suite """ -import os,shutil +import os, shutil from ..include.common import cfg -from .ct_base import CmdTestBase +from .base import CmdTestBase refdir = os.path.join('test','ref','ticker') diff --git a/test/cmdtest_d/ct_regtest.py b/test/cmdtest_d/regtest.py similarity index 98% rename from test/cmdtest_d/ct_regtest.py rename to test/cmdtest_d/regtest.py index f043641..3b77836 100755 --- a/test/cmdtest_d/ct_regtest.py +++ b/test/cmdtest_d/regtest.py @@ -9,18 +9,18 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.ct_regtest: Regtest tests for the cmdtest.py test suite +test.cmdtest_d.regtest: Regtest tests for the cmdtest.py test suite """ -import sys,os +import sys, os from decimal import Decimal -from mmgen.util import msg_r,die,gmsg +from mmgen.util import msg_r, die, gmsg from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest -from ..include.common import cfg,imsg,stop_test_daemons,joinpath -from .ct_base import CmdTestBase +from ..include.common import cfg, imsg, stop_test_daemons, joinpath +from .base import CmdTestBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] diff --git a/test/init.sh b/test/init.sh index 94671f9..134f4f3 100755 --- a/test/init.sh +++ b/test/init.sh @@ -81,23 +81,28 @@ create_dir_links() { delete_old_stuff() { rm -rf test/unit_tests.py + rm -rf test/cmdtest_d/common.py + rm -rf test/cmdtest_d/ct_base.py + rm -rf test/cmdtest_d/group_mgr.py + rm -rf test/cmdtest_d/runner.py } create_test_links() { paths=' - test/include symbolic - test/overlay/__init__.py symbolic - test/overlay/fakemods/mmgen symbolic - test/__init__.py symbolic - test/clean.py symbolic - test/cmdtest.py hard - test/modtest.py hard - test/test-release.sh symbolic - test/cmdtest_d/common.py symbolic - test/cmdtest_d/ct_base.py symbolic - test/cmdtest_d/runner.py symbolic - test/cmdtest_d/group_mgr.py symbolic - cmds/mmgen-regtest symbolic + test/include symbolic + test/overlay/__init__.py symbolic + test/overlay/fakemods/mmgen symbolic + test/__init__.py symbolic + test/clean.py symbolic + test/cmdtest.py hard + test/modtest.py hard + test/test-release.sh symbolic + test/cmdtest_d/base.py symbolic + test/cmdtest_d/include/common.py symbolic + test/cmdtest_d/include/runner.py symbolic + test/cmdtest_d/include/group_mgr.py symbolic + test/cmdtest_d/include/pexpect.py symbolic + cmds/mmgen-regtest symbolic ' while read path type; do [ "$path" ] || continue From ab93f54d541d4ee6511f9bf37e703d50fc2745f5 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 30 Mar 2025 10:18:52 +0000 Subject: [PATCH 44/81] update test suite for MMGen Wallet v15.1.dev26 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0602083..de6f13c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev9 +3.5.dev10 diff --git a/setup.cfg b/setup.cfg index 905e659..0ab2734 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev25 + mmgen-wallet>=15.1.dev26 pyyaml yahooquery From 106f201300d7dadb3a70e979db9373e9f6ca0e34 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 24 May 2025 10:10:24 +0000 Subject: [PATCH 45/81] update for MMGen Wallet v15.1.dev39 --- MANIFEST.in | 1 + mmgen_node_tools/BlocksInfo.py | 2 +- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 2 +- setup.cfg | 3 +-- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0443a99..77b53ac 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,5 +6,6 @@ include test/init.sh include test/test-release.d/*.sh include test/modtest_d/*.py include test/cmdtest_d/*.py +include test/cmdtest_d/include/cfg.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 26e6cfa..f19089a 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -26,7 +26,7 @@ from time import strftime,gmtime from decimal import Decimal from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int -from mmgen.rpc import json_encoder +from mmgen.rpc.util import json_encoder class RangeParser: diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index de6f13c..70beee3 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev10 +3.5.dev11 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 16591b8..e7b9952 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -103,7 +103,7 @@ class fee_bracket: def log(data,fn): import json - from mmgen.rpc import json_encoder + from mmgen.rpc.util import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( cfg = cfg, diff --git a/setup.cfg b/setup.cfg index 0ab2734..cd27742 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ project_urls = Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = - License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows Operating System :: MacOS @@ -38,7 +37,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev26 + mmgen-wallet>=15.1.dev39 pyyaml yahooquery From 5a24cbfc0a50d03203710c387763a1b893f3d5f3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 24 Sep 2025 10:45:50 +0000 Subject: [PATCH 46/81] Version 3.5.0 - released concurrently with MMGen Wallet v16.0.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 70beee3..1545d96 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev11 +3.5.0 diff --git a/setup.cfg b/setup.cfg index cd27742..9bca61c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev39 + mmgen-wallet==16.0.0 pyyaml yahooquery From cab8be016707b6886cbacf56666611b9a3fdad98 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:37 +0000 Subject: [PATCH 47/81] pyproject.toml: add ruff data --- pyproject.toml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 81515d2..25af44f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,41 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.ruff] +line-length = 106 +indent-width = 4 + +[tool.ruff.format] +quote-style = "single" +indent-style = "tab" + +[tool.ruff.lint] +ignore = [ + "E401", # multiple imports per line + "E701", # multiple statements per line + "E721", # use isinstance() + "E731", # lambda instead of def + "E402", # module import not top of file + "E722", # bare except + "E713", # membership 'not in' + "E741", # ambiguous variable name +] + +[tool.ruff.lint.per-file-ignores] +"test/include/common.py" = [ "F821" ] # undefined name 'cfg' +"test/misc/input_func.py" = [ "F401" ] # imported but unused +"test/modtest_d/cashaddr.py" = [ "F841" ] # assigned to but never used +"test/modtest_d/dep.py" = [ "F401" ] # imported but unused +"test/modtest_d/testdep.py" = [ "F401" ] # imported but unused +"test/modtest_d/obj.py" = [ "F841" ] # assigned to but never used +"test/objtest_d/*" = [ "F401" ] # imported but unused +"test/objattrtest_d/*" = [ "F401" ] # imported but unused +"test/overlay/fakemods/*" = [ "F403", "F405" ] # `import *` used +"test/*.py" = [ "F401" ] # imported but unused +"test/colortest.py" = [ "F403", "F405" ] # `import *` used +"test/tooltest2.py" = [ "F403", "F405" ] # `import *` used +"test/overlay/tree/*" = [ "ALL" ] + [tool.pylint.format] indent-string = "\t" indent-after-paren = 2 From 5c9d7053017f463193b9b5aa9dfb036f1844d697 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 48/81] test suite: lint, whitespace --- test/cmdtest_d/main.py | 2 +- test/cmdtest_d/misc.py | 4 ++-- test/cmdtest_d/regtest.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/cmdtest_d/main.py b/test/cmdtest_d/main.py index 6e35b7d..f1881f8 100755 --- a/test/cmdtest_d/main.py +++ b/test/cmdtest_d/main.py @@ -41,7 +41,7 @@ class CmdTestMain(CmdTestBase): def peerblocks(self,args,expect_list=None,pexpect_spawn=False): t = self.spawn( - f'mmnode-peerblocks', + 'mmnode-peerblocks', args, pexpect_spawn = pexpect_spawn ) if cfg.exact_output: # disable echoing of input diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 1a8175e..8a39462 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -32,7 +32,7 @@ class CmdTestHelp(CmdTestBase): color = True def version(self): - t = self.spawn(f'mmnode-netrate',['--version']) + t = self.spawn('mmnode-netrate', ['--version']) t.expect('MMNODE-NETRATE version') return t @@ -108,7 +108,7 @@ class CmdTestScripts(CmdTestBase): def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): t = self.spawn( - f'mmnode-ticker', + 'mmnode-ticker', (['--cached-data'] if cached else []) + self.ticker_args + args, exit_val = exit_val) if expect_list: diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index 3b77836..4792dee 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -107,7 +107,7 @@ class CmdTestRegtest(CmdTestBase): def __init__(self, cfg, trunner, cfgs, spawn): CmdTestBase.__init__(self, cfg, trunner, cfgs, spawn) - if trunner == None: + if trunner is None: return if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') @@ -328,7 +328,7 @@ class CmdTestRegtest(CmdTestBase): async def do_tx(inputs,outputs,wif): tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) - assert tx['complete'] == True + assert tx['complete'] return tx['hex'] async def do_tx1(): @@ -371,13 +371,13 @@ class CmdTestRegtest(CmdTestBase): imsg(f'Creating funding transaction with {nTxs} outputs of value {tx1_amt} {self.proto.coin}') tx1_hex = await do_tx1() - imsg(f'Relaying funding transaction') + imsg('Relaying funding transaction') await r.rpc_call('sendrawtransaction',tx1_hex) - imsg(f'Mining a block') + imsg('Mining a block') await r.generate(1,silent=True) - imsg(f'Generating fees for mempool transactions') + imsg('Generating fees for mempool transactions') fees = list(gen_fees(nTxs,2,120)) imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') From 4d1f4577a7758c0c028166d7f59538ec53d74d8d Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 49/81] lint, whitespace --- mmgen_node_tools/BlocksInfo.py | 2 +- mmgen_node_tools/PollDisplay.py | 2 +- mmgen_node_tools/Sound.py | 2 +- mmgen_node_tools/Ticker.py | 23 ++++++++++++--------- mmgen_node_tools/Util.py | 1 - mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 5 ++--- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index f19089a..d5eac9e 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -361,7 +361,7 @@ class BlocksInfo: if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) if nblocks: - if first == None: + if first is None: first = self.tip - nblocks + 1 last = first + nblocks - 1 diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 72bedac..1de8a1a 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -52,7 +52,7 @@ class PollDisplay: count += 1 async def process_input(): - if self.input == None: + if self.input is None: sys.exit(1) elif self.input == 'q': msg('') diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index 88d3b02..df9f8ae 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -33,7 +33,7 @@ def timespec2secs(ts): mul = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24 } pat = r'^([0-9]+)([smhd]*)$' m = re.match(pat,ts) - if m == None: + if m is None: die(2,"'%s': invalid time specifier" % ts) a,b = m.groups() return int(a) * (mul[b] if b else 1) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 252ba6c..6c8bc08 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -19,13 +19,13 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,re,time,datetime,json,yaml,random -from subprocess import run,PIPE,CalledProcessError +import sys, os, re, time, datetime, json, yaml, random +from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red,yellow,green,blue,orange,gray -from mmgen.util import msg,msg_r,Msg,Msg_r,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.color import red, yellow, green, blue, orange, gray +from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -89,9 +89,11 @@ class DataSource: msg('') from .Misc import curl_exit_codes msg(red(curl_exit_codes[e.returncode])) - msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) + msg(red('Command line:\n {}'.format( + ' '.join((repr(i) if ' ' in i else i) for i in e.cmd)))) from mmgen.exception import MMGenCalledProcessError - raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') + raise MMGenCalledProcessError( + f'Subprocess returned non-zero exit status {e.returncode}') def get_data(self): @@ -327,15 +329,16 @@ class DataSource: def postprocess_data(self,data): def gen(): keys = set() - for key,val in data.items(): + d = {} + for key, val in data.items(): if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) if (sym := m[1]) in keys: d[date] = val else: keys.add(sym) - d = {date:val} - yield (sym,d) + d = {date: val} + yield (sym, d) return dict(gen()) def assets_list_gen(cfg_in): @@ -936,7 +939,7 @@ class Ticker: def fmt_row(self,d,amt=None,amt_fmt=None): def fmt_pct(n): - return gray(' --') if n == None else (red,green)[n>=0](f'{n:+7.2f}') + return gray(' --') if n is None else (red,green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 9060f68..1d5426b 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -20,7 +20,6 @@ mmgen_node_tools.Util: utility functions for MMGen node tools """ import time -from mmgen.util import suf def get_hms(t=None,utc=False,no_secs=False): secs = t or time.time() diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index ecf2b1a..747d0bb 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -14,7 +14,7 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain import sys -from mmgen.obj import CoinTxID,Int +from mmgen.obj import CoinTxID from mmgen.cfg import Config from mmgen.util import msg,Msg,die,suf,make_timestr,async_run from mmgen.color import red diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index caeb178..50916ab 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -21,7 +21,6 @@ mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) """ import time -from decimal import Decimal from mmgen.cfg import Config from mmgen.util import async_run @@ -148,8 +147,8 @@ async def main(): b = 'BLOCK', c = 'DATE', d = '', - e = f'BDI (mins)', - f = f'SUBSIDY ({proto.coin})', + e = 'BDI (mins)', + f = 'SUBSIDY ({proto.coin})', g = f'MINED ({proto.coin})', h = f'TOTAL MINED ({proto.coin})' ) From bfb2dd839aab1d221e38d6e6532dca20f2904632 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 50/81] setup.cfg: bump requirements --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9bca61c..9af335b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,11 +33,11 @@ classifiers = Development Status :: 5 - Production/Stable [options] -python_requires = >=3.9 +python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet==16.0.0 + mmgen-wallet>=16.0.0 pyyaml yahooquery From 37c74e361c0f695a43b3f08b4b85cc2106b92665 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 51/81] use match statement where practicable --- mmgen_node_tools/BlocksInfo.py | 102 ++++++++++++++++++--------------- mmgen_node_tools/Ticker.py | 21 +++---- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index d5eac9e..42ccc23 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -197,18 +197,18 @@ class BlocksInfo: return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') def parse_cmd_args(): # => (block_list, first, last, step) - if not cmd_args: - return (None,self.tip,self.tip,None) - elif len(cmd_args) == 1: - r = self.parse_rangespec(cmd_args[0]) - return ( - list(range(r.first,r.last+1,r.step)) if r.step else None, - r.first, - r.last, - r.step - ) - else: - return ([self.conv_blkspec(a) for a in cmd_args],None,None,None) + match cmd_args: + case [] | None: + return (None, self.tip, self.tip, None) + case [arg]: + r = self.parse_rangespec(arg) + return ( + list(range(r.first, r.last+1, r.step)) if r.step else None, + r.first, + r.last, + r.step) + case [*args]: + return ([self.conv_blkspec(a) for a in args], None, None, None) self.cfg = cfg self.rpc = rpc @@ -323,25 +323,29 @@ class BlocksInfo: repl = (name if add_name else '') + ':' + (fill_char if name in fill else '') yield (ls + self.fields[name].fs.replace(':',repl) + rs) - def conv_blkspec(self,arg): - if str(arg).lower() == 'cur': - return self.tip - elif is_int(arg): - if int(arg) < 0: - die(1,f'{arg}: block number must be non-negative') - elif int(arg) > self.tip: - die(1,f'{arg}: requested block height greater than current chain tip!') - else: - return int(arg) - else: - die(1,f'{arg}: invalid block specifier') + def conv_blkspec(self, arg): + match arg: + case str() if arg.lower() == 'cur': + return self.tip + case x if is_int(x): + match int(arg): + case x if x < 0: + die(1, f'{x}: block number must be non-negative') + case x if x > self.tip: + die(1, f'{x}: requested block height greater than current chain tip!') + case x: + return x + case _: + die(1, f'{arg}: invalid block specifier') - def check_nblocks(self,arg): - if arg <= 0: - die(1,'nBlocks must be a positive integer') - elif arg > self.tip: - die(1, f"'{arg}': nBlocks must be less than current chain height") - return arg + def check_nblocks(self, arg): + match arg: + case x if x <= 0: + die(1, 'nBlocks must be a positive integer') + case x if x > self.tip: + die(1, f'{arg}: nBlocks must be less than current chain height') + case _: + return arg def parse_rangespec(self,arg): @@ -487,20 +491,21 @@ class BlocksInfo: def fmt_stat_item(self,fs,s): return fs.format(s) if type(fs) == str else fs(s) - async def output_stats(self,res,sname): + async def output_stats(self, res, sname): def gen(data): for d in data: - if len(d) == 2: - yield (indent+d[0]).format(**{k:self.fmt_stat_item(*v) for k,v in d[1].items()}) - elif len(d) == 4: - yield (indent+d[0]).format(self.fmt_stat_item(d[2],d[3])) - elif type(d) == str: - yield d - else: - assert False, f'{d}: invalid stats data' + match d: + case [a, b]: + yield (indent + a).format(**{k:self.fmt_stat_item(*v) for k, v in b.items()}) + case [a, _, b, c]: + yield (indent + a).format(self.fmt_stat_item(b, c)) + case str(): + yield d + case _: + assert False, f'{d}: invalid stats data' - foo,data = await res + foo, data = await res indent = '' if sname in self.noindent_stats else ' ' Msg('\n'.join(gen(data))) @@ -755,13 +760,16 @@ class JSONBlocksInfo(BlocksInfo): def gen(data): for d in data: - if len(d) == 2: - for k,v in d[1].items(): - yield (k,self.fmt_stat_item(*v)) - elif len(d) == 4: - yield (d[1],self.fmt_stat_item(d[2],d[3])) - elif type(d) != str: - assert False, f'{d}: invalid stats data' + match d: + case [_, a]: + for k, v in a.items(): + yield (k, self.fmt_stat_item(*v)) + case [_, a, b, c]: + yield (a, self.fmt_stat_item(b, c)) + case str(): + pass + case _: + assert False, f'{d}: invalid stats data' varname,data = await res Msg_r(', "{}_data": {}'.format( varname, json.dumps(dict(gen(data)),cls=json_encoder) )) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6c8bc08..95e03f2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -125,16 +125,17 @@ class DataSource: else: die(1,self.rate_limit_errmsg(elapsed)) - if data_type == 'json': - try: - data = json.loads(data_in) - except: - self.json_data_error_msg(data_in) - die(2,'Retrieved data is not valid JSON, exiting') - json_text = data_in - elif data_type == 'python': - data = data_in - json_text = json.dumps(data_in) + match data_type: + case 'json': + try: + data = json.loads(data_in) + except: + self.json_data_error_msg(data_in) + die(2,'Retrieved data is not valid JSON, exiting') + json_text = data_in + case 'python': + data = data_in + json_text = json.dumps(data_in) if not data: if use_cached_data: From fe45fbaa239ae6c28ed056cd1c2f0977550bad45 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 52/81] minor cleanups --- mmgen_node_tools/main_blocks_info.py | 4 ++-- mmgen_node_tools/main_halving_calculator.py | 4 ++-- test/modtest_d/__init__.py | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100755 test/modtest_d/__init__.py diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index c2bbf59..18136c4 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -164,7 +164,7 @@ async def main(): cls = JSONBlocksInfo if cfg.json else BlocksInfo - m = cls( cfg, cfg._args, await rpc_init(cfg,ignore_wallet=True) ) + m = cls(cfg, cfg._args, await rpc_init(cfg, ignore_wallet=True)) if m.fnames and not cfg.no_header: m.print_header() @@ -178,4 +178,4 @@ async def main(): m.finalize_output() -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 50916ab..8f0a3a3 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -67,7 +67,7 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init( cfg, proto, ignore_wallet=True ) + c = await rpc_init(cfg, proto, ignore_wallet=True) tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' @@ -181,4 +181,4 @@ async def main(): else: print_current_stats() -async_run(main()) +async_run(cfg, main) diff --git a/test/modtest_d/__init__.py b/test/modtest_d/__init__.py new file mode 100755 index 0000000..86190b5 --- /dev/null +++ b/test/modtest_d/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +""" +test.modtest_d: shared data for module tests for the MMGen Node Tools suite +""" + +altcoin_tests = [] From 1f12baac4c246f812942e4817dfd8541d5ee8913 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 53/81] update for MMGen Wallet v16.1.dev3 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_netrate.py | 2 +- mmgen_node_tools/main_peerblocks.py | 8 ++++---- mmgen_node_tools/main_txfind.py | 2 +- setup.cfg | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1545d96..229c4d8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.0 +3.6.dev0 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 747d0bb..37f0482 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -167,6 +167,6 @@ if len(cfg._args) < 1: die(1,'This command requires at least one coin address argument') try: - async_run(main(cfg._args)) + async_run(cfg, main, args=[cfg._args]) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index e7b9952..3211f7f 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -220,4 +220,4 @@ async def main(): await c.call('getblockcount') )) + '\n\n' + '\n'.join(gen_body(data)) + '\n' ) -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index 52b509d..b273ac6 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -67,6 +67,6 @@ async def main(): rs,ss,ts = (r,s,t) try: - async_run(main()) + async_run(cfg, main) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 6ebb5a3..35292ea 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -31,10 +31,10 @@ opts_data = { } } -async def main(): +from mmgen.cfg import Config +cfg = Config(opts_data=opts_data) - from mmgen.cfg import Config - cfg = Config(opts_data=opts_data) +async def main(): from mmgen.rpc import rpc_init rpc = await rpc_init(cfg,ignore_wallet=True) @@ -48,4 +48,4 @@ async def main(): await peers.run(rpc) from mmgen.util import async_run -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 4a54430..2411f36 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -92,4 +92,4 @@ msgs = msg_data['quiet' if cfg.quiet else 'normal'] if len(cfg._args) != 1: die(1,'One transaction ID must be specified') -sys.exit(async_run(main(cfg._args[0]))) +sys.exit(async_run(cfg, main, args=[cfg._args[0]])) diff --git a/setup.cfg b/setup.cfg index 9af335b..29481b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.0.0 + mmgen-wallet>=16.1.dev3 pyyaml yahooquery From 0bad23b77bd2327937ec82ef7e8d4dba127e44be Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 3 Oct 2025 10:31:49 +0000 Subject: [PATCH 54/81] update for MMGen Wallet v16.1.dev4 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/include/cfg.py | 12 ++++++++---- test/cmdtest_d/misc.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 229c4d8..9f32e9d 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev0 +3.6.dev1 diff --git a/setup.cfg b/setup.cfg index 29481b2..d48a8c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev3 + mmgen-wallet>=16.1.dev4 pyyaml yahooquery diff --git a/test/cmdtest_d/include/cfg.py b/test/cmdtest_d/include/cfg.py index 5a4263d..08b6263 100755 --- a/test/cmdtest_d/include/cfg.py +++ b/test/cmdtest_d/include/cfg.py @@ -12,13 +12,17 @@ test.cmdtest_d.include.cfg: configuration data for cmdtest.py """ +from collections import namedtuple + cmd_groups_altcoin = [] +gd = namedtuple('cmd_groups_data', ['clsname', 'params']) + cmd_groups_dfl = { - 'main': ('CmdTestMain',{}), - 'helpscreens': ('CmdTestHelp',{'modname':'misc','full_data':True}), - 'scripts': ('CmdTestScripts',{'modname':'misc'}), - 'regtest': ('CmdTestRegtest',{}), + 'main': gd('CmdTestMain', {}), + 'helpscreens': gd('CmdTestHelp', {'modname': 'misc', 'full_data': True}), + 'scripts': gd('CmdTestScripts', {'modname': 'misc'}), + 'regtest': gd('CmdTestRegtest', {}), } cmd_groups_extra = {} diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 8a39462..9528b76 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -125,7 +125,7 @@ class CmdTestScripts(CmdTestBase): if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - ret = t.expect(['proxy host could not be resolved', 'ProxyError']) + ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) t.exit_val = 1 if ret else 3 return t From 0b79ef719b3712b64571b63dbd3177ab247aa8b3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 09:56:55 +0000 Subject: [PATCH 55/81] whitespace, minor changes (16 files) --- mmgen_node_tools/BlocksInfo.py | 402 +++++++-------- mmgen_node_tools/PeerBlocks.py | 55 +- mmgen_node_tools/PollDisplay.py | 12 +- mmgen_node_tools/Sound.py | 32 +- mmgen_node_tools/Ticker.py | 485 ++++++++---------- mmgen_node_tools/Util.py | 60 ++- mmgen_node_tools/main_addrbal.py | 66 ++- mmgen_node_tools/main_blocks_info.py | 19 +- mmgen_node_tools/main_feeview.py | 65 ++- mmgen_node_tools/main_halving_calculator.py | 56 +- mmgen_node_tools/main_netrate.py | 21 +- mmgen_node_tools/main_peerblocks.py | 7 +- mmgen_node_tools/main_ticker.py | 23 +- mmgen_node_tools/main_txfind.py | 17 +- test/cmdtest_d/regtest.py | 8 +- .../fakemods/mmgen_node_tools/PeerBlocks.py | 8 +- 16 files changed, 629 insertions(+), 707 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 42ccc23..a59bf50 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -20,58 +20,57 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of blocks """ -import re,json +import re, json from collections import namedtuple -from time import strftime,gmtime +from time import strftime, gmtime from decimal import Decimal -from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int +from mmgen.util import msg, Msg, Msg_r, die, suf, secs_to_ms, secs_to_dhms, is_int from mmgen.rpc.util import json_encoder class RangeParser: debug = False - def __init__(self,caller,arg): + def __init__(self, caller, arg): self.caller = caller self.arg = self.orig_arg = arg - def parse(self,target): - ret = getattr(self,'parse_'+target)() + def parse(self, target): + ret = getattr(self, 'parse_'+target)() if self.debug: msg(f'arg after parse({target}): {self.arg}') return ret def finalize(self): if self.arg: - die(1,f'{self.orig_arg!r}: invalid range specifier') + die(1, f'{self.orig_arg!r}: invalid range specifier') def parse_from_tip(self): - m = re.match(r'-([0-9]+)(.*)',self.arg) + m = re.match(r'-([0-9]+)(.*)', self.arg) if m: - res,self.arg = (m[1],m[2]) + res, self.arg = (m[1], m[2]) return self.caller.check_nblocks(int(res)) def parse_abs_range(self): - m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',self.arg) + m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)', self.arg) if m: if self.debug: msg(f'abs_range parse: first={m[1]}, last={m[3]}') self.arg = m[4] return ( self.caller.conv_blkspec(m[1]), - self.caller.conv_blkspec(m[3]) if m[3] else None - ) - return (None,None) + self.caller.conv_blkspec(m[3]) if m[3] else None) + return (None, None) def parse_add(self): - m = re.match(r'\+([0-9*]+)(.*)',self.arg) + m = re.match(r'\+([0-9*]+)(.*)', self.arg) if m: - res,self.arg = (m[1],m[2]) + res, self.arg = (m[1], m[2]) if res.strip('*') != res: - die(1,f"'+{res}': malformed nBlocks specifier") + die(1, f"'+{res}': malformed nBlocks specifier") if len(res) > 30: - die(1,f"'+{res}': overly long nBlocks specifier") + die(1, f"'+{res}': overly long nBlocks specifier") return self.caller.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe class BlocksInfo: @@ -81,33 +80,33 @@ class BlocksInfo: total_solve_time = 0 header_printed = False - bf = namedtuple('block_info_fields',['fmt_func','src','fs','hdr1','hdr2','key1','key2']) + bf = namedtuple('block_info_fields', ['fmt_func', 'src', 'fs', 'hdr1', 'hdr2', 'key1', 'key2']) # bh=getblockheader, bs=getblockstats, lo=local fields = { - 'block': bf( None, 'bh', '{:<6}', '', 'Block', 'height', None ), - 'hash': bf( None, 'bh', '{:<64}', '', 'Hash', 'hash', None ), - 'date': bf( 'da', 'bh', '{:<19}', '', 'Date', 'time', None ), - 'interval': bf( 'td', 'lo', '{:>8}', 'Solve', 'Time ', 'interval', None ), - 'subsidy': bf( 'su', 'bs', '{:<5}', 'Sub-', 'sidy', 'subsidy', None ), - 'totalfee': bf( 'tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None ), - 'size': bf( None, 'bs', '{:>7}', '', 'Size', 'total_size', None ), - 'weight': bf( None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None ), - 'fee90': bf( 'fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4 ), - 'fee75': bf( 'fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3 ), - 'fee50': bf( 'fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2 ), - 'fee25': bf( 'fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1 ), - 'fee10': bf( 'fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0 ), - 'fee_max': bf( 'fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None ), - 'fee_avg': bf( 'fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None ), - 'fee_min': bf( 'fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None ), - 'nTx': bf( None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None ), - 'inputs': bf( None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None ), - 'outputs': bf( None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None ), - 'utxo_inc': bf( None, 'bs', '{:>6}', ' UTXO', ' Incr', 'utxo_increase', None ), - 'version': bf( None, 'bh', '{:<8}', '', 'Version', 'versionHex', None ), - 'difficulty': bf( 'di', 'bh', '{:<8}', 'Diffi-','culty', 'difficulty', None ), - 'miner': bf( None, 'lo', '{:<5}', '', 'Miner', 'miner', None ), - } + 'block': bf(None, 'bh', '{:<6}', '', 'Block', 'height', None), + 'hash': bf(None, 'bh', '{:<64}', '', 'Hash', 'hash', None), + 'date': bf('da', 'bh', '{:<19}', '', 'Date', 'time', None), + 'interval': bf('td', 'lo', '{:>8}', 'Solve', 'Time ', 'interval', None), + 'subsidy': bf('su', 'bs', '{:<5}', 'Sub-', 'sidy', 'subsidy', None), + 'totalfee': bf('tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None), + 'size': bf(None, 'bs', '{:>7}', '', 'Size', 'total_size', None), + 'weight': bf(None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None), + 'fee90': bf('fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4), + 'fee75': bf('fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3), + 'fee50': bf('fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2), + 'fee25': bf('fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1), + 'fee10': bf('fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0), + 'fee_max': bf('fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None), + 'fee_avg': bf('fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None), + 'fee_min': bf('fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None), + 'nTx': bf(None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None), + 'inputs': bf(None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None), + 'outputs': bf(None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None), + 'utxo_inc': bf(None, 'bs', '{:>6}', ' UTXO', ' Incr', 'utxo_increase', None), + 'version': bf(None, 'bh', '{:<8}', '', 'Version', 'versionHex', None), + 'difficulty': bf('di', 'bh', '{:<8}', 'Diffi-','culty', 'difficulty', None), + 'miner': bf(None, 'lo', '{:<5}', '', 'Miner', 'miner', None)} + dfl_fields = ( 'block', 'date', @@ -121,8 +120,8 @@ class BlocksInfo: 'fee10', 'fee_avg', 'fee_min', - 'version', - ) + 'version') + fixed_fields = ( 'block', # until ≈ 09/01/2028 (block 1000000) 'hash', @@ -131,36 +130,34 @@ class BlocksInfo: 'weight', # until ≈ 2.5x block size increase 'version', 'subsidy', # until ≈ 01/04/2028 (increases by 1 digit per halving until 9th halving [max 10 digits]) - 'difficulty', # until 1.00e+100 (i.e. never) - ) + 'difficulty') # until 1.00e+100 (i.e. never) # column width adjustment data: - fs_lsqueeze = ('totalfee','inputs','outputs','nTx') + fs_lsqueeze = ('totalfee', 'inputs', 'outputs', 'nTx') fs_rsqueeze = () fs_groups = ( - ('fee10','fee25','fee50','fee75','fee90','fee_avg','fee_min','fee_max'), - ) + ('fee10', 'fee25', 'fee50', 'fee75', 'fee90', 'fee_avg', 'fee_min', 'fee_max')) fs_lsqueeze2 = ('interval',) - all_stats = ['col_avg','range','avg','mini_avg','total','diff'] - dfl_stats = ['range','mini_avg','diff'] + all_stats = ['col_avg', 'range', 'avg', 'mini_avg', 'total', 'diff'] + dfl_stats = ['range', 'mini_avg', 'diff'] noindent_stats = ['col_avg'] - avg_stats_skip = {'block', 'hash', 'date', 'version','miner'} + avg_stats_skip = {'block', 'hash', 'date', 'version', 'miner'} - range_data = namedtuple('parsed_range_data',['first','last','from_tip','nblocks','step']) + range_data = namedtuple('parsed_range_data', ['first', 'last', 'from_tip', 'nblocks', 'step']) - t_fmt = lambda self,t: f'{t/86400:.2f} days' if t > 172800 else f'{t/3600:.2f} hrs' + t_fmt = lambda self, t: f'{t/86400:.2f} days' if t > 172800 else f'{t/3600:.2f} hrs' @classmethod - def parse_cslist(cls,uarg,full_set,dfl_set,desc): + def parse_cslist(cls, uarg, full_set, dfl_set, desc): - def make_list(m,func): + def make_list(m, func): groups_lc = [set(e.lower() for e in gi.split(',')) for gi in m.groups()] for group in groups_lc: for e in group: if e not in full_set_lc: - die(1,f'{e!r}: unrecognized {desc}') + die(1, f'{e!r}: unrecognized {desc}') # display elements in order: return [e for e in full_set if e.lower() in func(groups_lc)] @@ -168,33 +165,31 @@ class BlocksInfo: dfl_set_lc = set(e.lower() for e in dfl_set) cspat = r'(\w+(?:,\w+)*)' - for pat,func in ( - ( rf'{cspat}$', lambda g: g[0] ), - ( rf'\+{cspat}$', lambda g: dfl_set_lc | g[0] ), - ( rf'\-{cspat}$', lambda g: dfl_set_lc - g[0] ), - ( rf'\+{cspat}\-{cspat}$', lambda g: ( dfl_set_lc | g[0] ) - g[1] ), - ( rf'\-{cspat}\+{cspat}$', lambda g: ( dfl_set_lc - g[0] ) | g[1] ), - ( rf'all\-{cspat}$', lambda g: full_set_lc - g[0] ) - ): - m = re.match(pat,uarg,re.ASCII|re.IGNORECASE) + for pat, func in ( + (rf'{cspat}$', lambda g: g[0]), + (rf'\+{cspat}$', lambda g: dfl_set_lc | g[0]), + (rf'\-{cspat}$', lambda g: dfl_set_lc - g[0]), + (rf'\+{cspat}\-{cspat}$', lambda g: (dfl_set_lc | g[0]) - g[1]), + (rf'\-{cspat}\+{cspat}$', lambda g: (dfl_set_lc - g[0]) | g[1]), + (rf'all\-{cspat}$', lambda g: full_set_lc - g[0])): + m = re.match(pat, uarg, re.ASCII|re.IGNORECASE) if m: - return make_list(m,func) + return make_list(m, func) else: - die(1,f'{uarg}: invalid parameter') + die(1, f'{uarg}: invalid parameter') - def __init__(self,cfg,cmd_args,rpc): + def __init__(self, cfg, cmd_args, rpc): - def parse_cs_uarg(uarg,full_set,dfl_set,desc): + def parse_cs_uarg(uarg, full_set, dfl_set, desc): return ( full_set if uarg == 'all' else [] if uarg == 'none' else - self.parse_cslist(uarg,full_set,dfl_set,desc) - ) + self.parse_cslist(uarg, full_set, dfl_set, desc)) def get_fields(): - return parse_cs_uarg(self.cfg.fields,list(self.fields),self.dfl_fields,'field') + return parse_cs_uarg(self.cfg.fields, list(self.fields), self.dfl_fields, 'field') def get_stats(): - return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') + return parse_cs_uarg(self.cfg.stats.lower(), self.all_stats, self.dfl_stats, 'stat') def parse_cmd_args(): # => (block_list, first, last, step) match cmd_args: @@ -217,7 +212,7 @@ class BlocksInfo: from_satoshi = self.rpc.proto.coin_amt.satoshi to_satoshi = 1 / from_satoshi - self.block_list,self.first,self.last,self.step = parse_cmd_args() + self.block_list, self.first, self.last, self.step = parse_cmd_args() have_segwit = self.rpc.info('segwit_is_active') @@ -228,35 +223,33 @@ class BlocksInfo: self.stats_deps = { 'avg': set(self.fields) - self.avg_stats_skip, 'col_avg': set(self.fields) - self.avg_stats_skip, - 'mini_avg': {'interval','size'} | ({'weight'} if have_segwit else set()), - 'total': {'interval','subsidy','totalfee','nTx','inputs','outputs','utxo_inc'}, + 'mini_avg': {'interval', 'size'} | ({'weight'} if have_segwit else set()), + 'total': {'interval', 'subsidy', 'totalfee', 'nTx', 'inputs', 'outputs', 'utxo_inc'}, 'range': {}, - 'diff': {}, - } + 'diff': {}} self.fmt_funcs = { - 'da': lambda arg: strftime('%Y-%m-%d %X',gmtime(arg)), + 'da': lambda arg: strftime('%Y-%m-%d %X', gmtime(arg)), 'td': lambda arg: ( - '-{:02}:{:02}'.format(abs(arg)//60,abs(arg)%60) if arg < 0 else - ' {:02}:{:02}'.format(arg//60,arg%60) ), + '-{:02}:{:02}'.format(abs(arg)//60, abs(arg)%60) if arg < 0 else + ' {:02}:{:02}'.format(arg//60, arg%60)), 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), 'fe': lambda arg: str(arg), - 'di': lambda arg: '{:.2e}'.format(Decimal(arg)), - } + 'di': lambda arg: '{:.2e}'.format(Decimal(arg))} if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), 'fe': lambda arg: str(int(Decimal(arg) * to_satoshi)), - 'tf': lambda arg: '{:.8f}'.format(Decimal(arg)), - }) + 'tf': lambda arg: '{:.8f}'.format(Decimal(arg))}) self.fnames = tuple( - [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] if self.cfg.header_info else - get_fields() if self.cfg.fields else - self.dfl_fields - ) + [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] + if self.cfg.header_info + else get_fields() if self.cfg.fields + else self.dfl_fields) + if self.cfg.miner_info and 'miner' not in self.fnames: self.fnames += ('miner',) @@ -266,15 +259,15 @@ class BlocksInfo: if 'diff' in self.stats and not self.cfg.stats and self.last != self.tip: self.stats.remove('diff') - if {'avg','col_avg'} <= set(self.stats) and self.cfg.stats_only: + if {'avg', 'col_avg'} <= set(self.stats) and self.cfg.stats_only: self.stats.remove('col_avg') - if {'avg','mini_avg'} <= set(self.stats): + if {'avg', 'mini_avg'} <= set(self.stats): self.stats.remove('mini_avg') if self.cfg.full_stats: add_fnames = {fname for sname in self.stats for fname in self.stats_deps[sname]} - self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames ) + self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames) else: if 'col_avg' in self.stats and not self.fnames: self.stats.remove('col_avg') @@ -287,8 +280,7 @@ class BlocksInfo: self.bs_keys = set( [v.key1 for v in self.fvals if v.src == 'bs'] + ['total_size'] + - (['total_weight'] if have_segwit else []) - ) + (['total_weight'] if have_segwit else [])) if 'miner' in self.fnames: # capturing parens must contain only ASCII chars! @@ -302,17 +294,16 @@ class BlocksInfo: rb'([\x20-\x7e]{9,})', rb'[/^]([a-zA-Z0-9&. #/-]{5,})', rb'[/^]([_a-zA-Z0-9&. #/-]+)/', - rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]', - )] + rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]')] - self.block_data = namedtuple('block_data',self.fnames) - self.deps = { v.src for v in self.fvals } + self.block_data = namedtuple('block_data', self.fnames) + self.deps = {v.src for v in self.fvals} - def gen_fs(self,fnames,fill=[],fill_char='-',add_name=False): + def gen_fs(self, fnames, fill=[], fill_char='-', add_name=False): for i in range(len(fnames)): name = fnames[i] - ls = (' ','')[name in self.fs_lsqueeze + self.fs_lsqueeze2] - rs = (' ','')[name in self.fs_rsqueeze] + ls = (' ', '')[name in self.fs_lsqueeze + self.fs_lsqueeze2] + rs = (' ', '')[name in self.fs_rsqueeze] if i < len(fnames) - 1 and fnames[i+1] in self.fs_lsqueeze2: rs = '' if i: @@ -321,7 +312,7 @@ class BlocksInfo: ls = '' break repl = (name if add_name else '') + ':' + (fill_char if name in fill else '') - yield (ls + self.fields[name].fs.replace(':',repl) + rs) + yield (ls + self.fields[name].fs.replace(':', repl) + rs) def conv_blkspec(self, arg): match arg: @@ -347,22 +338,22 @@ class BlocksInfo: case _: return arg - def parse_rangespec(self,arg): + def parse_rangespec(self, arg): - p = RangeParser(self,arg) + p = RangeParser(self, arg) - from_tip = p.parse('from_tip') - first,last = (self.tip-from_tip,None) if from_tip else p.parse('abs_range') - add1 = p.parse('add') - add2 = p.parse('add') + from_tip = p.parse('from_tip') + first, last = (self.tip-from_tip, None) if from_tip else p.parse('abs_range') + add1 = p.parse('add') + add2 = p.parse('add') p.finalize() if add2 and last is not None: - die(1,f'{arg!r}: invalid range specifier') + die(1, f'{arg!r}: invalid range specifier') - nblocks,step = (add1,add2) if last is None else (None,add1) + nblocks, step = (add1, add2) if last is None else (None, add1) - if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + if p.debug: msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if nblocks: if first is None: @@ -373,12 +364,12 @@ class BlocksInfo: last = self.conv_blkspec(last or first) if p.debug: - msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if first > last: - die(1,f'{first}-{last}: invalid block range') + die(1, f'{first}-{last}: invalid block range') - return self.range_data(first,last,from_tip,nblocks,step) + return self.range_data(first, last, from_tip, nblocks, step) async def process_blocks(self): @@ -388,7 +379,7 @@ class BlocksInfo: c = self.rpc - heights = self.block_list or range(self.first,self.last+1) + heights = self.block_list or range(self.first, self.last+1) self.hdrs = await get_hdrs(heights) if self.block_list: @@ -397,8 +388,7 @@ class BlocksInfo: else: self.first_prev_hdr = ( self.hdrs[0] if heights[0] == 0 else - await c.call('getblockheader',await c.call('getblockhash',heights[0]-1)) - ) + await c.call('getblockheader', await c.call('getblockhash', heights[0]-1))) self.t_cur = self.first_prev_hdr['time'] self.res = [] @@ -409,16 +399,16 @@ class BlocksInfo: ret = await self.process_block(self.hdrs[n]) self.res.append(ret) if self.fnames and not self.cfg.stats_only: - self.output_block(ret,n) + self.output_block(ret, n) - def output_block(self,data,n): + def output_block(self, data, n): def gen(): - for k,v in data._asdict().items(): + for k, v in data._asdict().items(): func = self.fields[k].fmt_func yield self.fmt_funcs[func](v) if func else v Msg(self.fs.format(*gen())) - async def process_block(self,hdr): + async def process_block(self, hdr): self.t_diff = hdr['time'] - self.t_cur self.t_cur = hdr['time'] @@ -426,14 +416,12 @@ class BlocksInfo: blk_data = { 'bh': hdr, - 'lo': { 'interval': self.t_diff } - } + 'lo': {'interval': self.t_diff}} if 'bs' in self.deps: bs = ( self.genesis_stats if hdr['height'] == 0 else - await self.rpc.call('getblockstats',hdr['hash'],list(self.bs_keys)) - ) + await self.rpc.call('getblockstats', hdr['hash'], list(self.bs_keys))) self.total_bytes += bs['total_size'] if 'total_weight' in bs: self.total_weight += bs['total_weight'] @@ -446,14 +434,13 @@ class BlocksInfo: for v in self.fvals: yield ( blk_data[v.src][v.key1] if v.key2 is None else - blk_data[v.src][v.key1][v.key2] - ) + blk_data[v.src][v.key1][v.key2]) return self.block_data(*gen()) - async def get_miner_string(self,H): - tx0 = (await self.rpc.call('getblock',H))['tx'][0] - bd = await self.rpc.call('getrawtransaction',tx0,1) + async def get_miner_string(self, H): + tx0 = (await self.rpc.call('getblock', H))['tx'][0] + bd = await self.rpc.call('getrawtransaction', tx0, 1) if type(bd) == tuple: return '---' else: @@ -464,13 +451,12 @@ class BlocksInfo: trmap_in = { '\\': ' ', '/': ' ', - ',': ' ', - } - trmap = { ord(a):b for a,b in trmap_in.items() } + ',': ' '} + trmap = {ord(a): b for a, b in trmap_in.items()} for pat in self.miner_pats: m = pat.search(cb) if m: - return re.sub( r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip() ) + return re.sub(r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip()) return '' def print_header(self): @@ -484,11 +470,11 @@ class BlocksInfo: yield self.fs.format(*hdr1) yield self.fs.format(*hdr2) - def process_stats(self,sname): - method = getattr(self,f'create_{sname}_stats',None) - return self.output_stats(method() if method else self.create_stats(sname),sname) + def process_stats(self, sname): + method = getattr(self, f'create_{sname}_stats', None) + return self.output_stats(method() if method else self.create_stats(sname), sname) - def fmt_stat_item(self,fs,s): + def fmt_stat_item(self, fs, s): return fs.format(s) if type(fs) == str else fs(s) async def output_stats(self, res, sname): @@ -497,7 +483,7 @@ class BlocksInfo: for d in data: match d: case [a, b]: - yield (indent + a).format(**{k:self.fmt_stat_item(*v) for k, v in b.items()}) + yield (indent + a).format(**{k: self.fmt_stat_item(*v) for k, v in b.items()}) case [a, _, b, c]: yield (indent + a).format(self.fmt_stat_item(b, c)) case str(): @@ -524,15 +510,14 @@ class BlocksInfo: 'range': ('{}', self.hdrs[-1]['height'] - self.hdrs[0]['height'] + 1), 'elapsed': (self.t_fmt, elapsed), 'nBlocks': ('{}', total_blks), - 'step': ('{}', self.step), - } - ) - if elapsed: - yield ( 'Start: {}', 'start_date', self.fmt_funcs['da'], self.hdrs[0]['time'] ) - yield ( 'End: {}', 'end_date', self.fmt_funcs['da'], self.hdrs[-1]['time'] ) - yield ( 'Avg BDI: {} min', 'avg_bdi', '{:.2f}', elapsed / nblocks / 60 ) + 'step': ('{}', self.step)}) - return ( 'range', gen() ) + if elapsed: + yield ('Start: {}', 'start_date', self.fmt_funcs['da'], self.hdrs[0]['time']) + yield ('End: {}', 'end_date', self.fmt_funcs['da'], self.hdrs[-1]['time']) + yield ('Avg BDI: {} min', 'avg_bdi', '{:.2f}', elapsed / nblocks / 60) + + return ('range', gen()) async def create_diff_stats(self): @@ -541,18 +526,17 @@ class BlocksInfo: tip_hdr = ( self.hdrs[-1] if self.hdrs[-1]['height'] == self.tip else - await c.call('getblockheader',await c.call('getblockhash',self.tip)) - ) + await c.call('getblockheader', await c.call('getblockhash', self.tip))) min_sample_blks = 432 # ≈3 days - rel_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-rel)) + rel_hdr = await c.call('getblockheader', await c.call('getblockhash', self.tip-rel)) if rel >= min_sample_blks: sample_blks = rel - bdi = ( tip_hdr['time'] - rel_hdr['time'] ) / rel + bdi = (tip_hdr['time'] - rel_hdr['time']) / rel else: - sample_blks = min(min_sample_blks,self.tip) - start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) + sample_blks = min(min_sample_blks, self.tip) + start_hdr = await c.call('getblockheader', await c.call('getblockhash', self.tip-sample_blks)) diff_adj = Decimal(tip_hdr['difficulty']) / Decimal(start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] @@ -560,7 +544,7 @@ class BlocksInfo: rem = self.rpc.proto.diff_adjust_interval - rel - return ( 'difficulty', ( + return ('difficulty', ( 'Difficulty Statistics:', ('Current height: {}', 'chain_tip', '{}', self.tip), ('Next diff adjust: {next_diff_adjust} (in {blks_remaining} block%s [{time_remaining}])' % suf(rem), @@ -593,105 +577,98 @@ class BlocksInfo: def gen(): for field in self.fnames: if field in self.avg_stats_skip: - yield ( field, ('{}','') ) + yield (field, ('{}', '')) else: ret = self.sum_field_avg(field) func = self.fields[field].fmt_func - yield ( field, ( (self.fmt_funcs[func] if func else '{}'), ret )) + yield (field, ((self.fmt_funcs[func] if func else '{}'), ret)) if not self.header_printed: self.print_header() - fs = ''.join(self.gen_fs(self.fnames,fill=self.avg_stats_skip,add_name=True)).strip() - return ('column_averages', ('Column averages:', (fs, dict(gen())) )) + fs = ''.join(self.gen_fs(self.fnames, fill=self.avg_stats_skip, add_name=True)).strip() + return ('column_averages', ('Column averages:', (fs, dict(gen())))) - def avg_stats_data(self,data,spec_conv,spec_val): + def avg_stats_data(self, data, spec_conv, spec_val): coin = self.rpc.proto.coin return data( hdr = 'Averages for processed blocks:', func = self.sum_field_avg, - spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}' }, + spec_sufs = {'subsidy': f' {coin}', 'totalfee': f' {coin}'}, spec_convs = { - 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), + 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'mb_per_hour': spec_conv(0, '{}'), - }, + 'mb_per_hour': spec_conv(0, '{}')}, spec_vals = ( spec_val( 'mb_per_hour', 'MB/hr', 'interval', lambda values: 'bs' in self.deps, lambda values: ( '{:.4f}'.format((self.total_bytes / 10000) / (self.total_solve_time / 36)) - if self.total_solve_time else 'N/A' ), - ), - ) - ) + if self.total_solve_time else 'N/A')), + )) mini_avg_stats_data = avg_stats_data - def total_stats_data(self,data,spec_conv,spec_val): + def total_stats_data(self, data, spec_conv, spec_val): coin = self.rpc.proto.coin return data( hdr = 'Totals for processed blocks:', func = self.sum_field_total, - spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}' }, + spec_sufs = {'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}'}, spec_convs = { - 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), + 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'reward': spec_conv(0, self.fmt_funcs['tf']), - }, + 'reward': spec_conv(0, self.fmt_funcs['tf'])}, spec_vals = ( spec_val( 'reward', 'Reward', 'totalfee', - lambda values: {'subsidy','totalfee'} <= set(values), - lambda values: values['subsidy'] + values['totalfee'] - ), - ) - ) + lambda values: {'subsidy', 'totalfee'} <= set(values), + lambda values: values['subsidy'] + values['totalfee']), + )) - async def create_stats(self,sname): + async def create_stats(self, sname): def convert_stats_hdr(field): v = self.fields[field] - return '{} {}'.format(v.hdr1.strip(), v.hdr2.strip()).replace('- ','') if v.hdr1 else v.hdr2.strip() + return '{} {}'.format( + v.hdr1.strip(), v.hdr2.strip()).replace('- ', '') if v.hdr1 else v.hdr2.strip() - d = getattr(self,f'{sname}_stats_data')( - namedtuple('stats_data',['hdr','func','spec_sufs','spec_convs','spec_vals']), - namedtuple('spec_conv',['width_adj','conv']), - namedtuple('spec_val',['name','lbl','insert_after','condition','code']) - ) + d = getattr(self, f'{sname}_stats_data')( + namedtuple('stats_data', ['hdr', 'func', 'spec_sufs', 'spec_convs', 'spec_vals']), + namedtuple('spec_conv', ['width_adj', 'conv']), + namedtuple('spec_val', ['name', 'lbl', 'insert_after', 'condition', 'code'])) fnames = [n for n in self.fnames if n in self.stats_deps[sname]] - lbls = {n:convert_stats_hdr(n) for n in fnames} - values = {n:d.func(n) for n in fnames} - col1_w = max((len(l) for l in lbls.values()),default=0) + 2 + lbls = {n: convert_stats_hdr(n) for n in fnames} + values = {n: d.func(n) for n in fnames} + col1_w = max((len(l) for l in lbls.values()), default=0) + 2 + print(d.spec_vals) for v in d.spec_vals: + print(v) if v.condition(values): try: idx = fnames.index(v.insert_after) + 1 except: idx = 0 - fnames.insert(idx,v.name) + fnames.insert(idx, v.name) lbls[v.name] = v.lbl values[v.name] = v.code(values) def gen(): - for n,fname in enumerate(fnames): + for n, fname in enumerate(fnames): spec_conv = d.spec_convs.get(fname) yield ( '{lbl:{wid}} {{}}{suf}'.format( lbl = lbls[fname] + ':', wid = col1_w + (spec_conv.width_adj if spec_conv else 0), - suf = d.spec_sufs.get(fname) or '' - ), + suf = d.spec_sufs.get(fname) or ''), fname, spec_conv.conv if spec_conv else ( - (lambda x: self.fmt_funcs[x] if x else '{}')(self.fields[fname].fmt_func) - ), - values[fname] - ) + (lambda x: self.fmt_funcs[x] if x else '{}')(self.fields[fname].fmt_func)), + values[fname]) - return ( sname, (d.hdr,) + tuple(gen()) ) + return (sname, (d.hdr,) + tuple(gen())) - def process_stats_pre(self,i): + def process_stats_pre(self, i): if (self.fnames and not self.cfg.stats_only) or i != 0: Msg('') @@ -724,13 +701,12 @@ class BlocksInfo: 'totalfee': 0, 'txs': 1, 'utxo_increase': 1, - 'utxo_size_inc': 117 - } + 'utxo_size_inc': 117} class JSONBlocksInfo(BlocksInfo): - def __init__(self,cfg,cmd_args,rpc): - super().__init__(cfg,cmd_args,rpc) + def __init__(self, cfg, cmd_args, rpc): + super().__init__(cfg, cmd_args, rpc) if self.cfg.json_raw: self.output_block = self.output_block_raw self.fmt_stat_item = self.fmt_stat_item_raw @@ -741,22 +717,22 @@ class JSONBlocksInfo(BlocksInfo): await super().process_blocks() Msg_r(']') - def output_block_raw(self,data,n): - Msg_r( (', ','')[n==0] + json.dumps(data._asdict(),cls=json_encoder) ) + def output_block_raw(self, data, n): + Msg_r((', ', '')[n==0] + json.dumps(data._asdict(), cls=json_encoder)) - def output_block(self,data,n): + def output_block(self, data, n): def gen(): - for k,v in data._asdict().items(): + for k, v in data._asdict().items(): func = self.fields[k].fmt_func - yield ( k, (self.fmt_funcs[func](v) if func else v) ) - Msg_r( (', ','')[n==0] + json.dumps(dict(gen()),cls=json_encoder) ) + yield (k, (self.fmt_funcs[func](v) if func else v)) + Msg_r((', ', '')[n==0] + json.dumps(dict(gen()), cls=json_encoder)) def print_header(self): pass - def fmt_stat_item_raw(self,fs,s): + def fmt_stat_item_raw(self, fs, s): return s - async def output_stats(self,res,sname): + async def output_stats(self, res, sname): def gen(data): for d in data: @@ -771,10 +747,10 @@ class JSONBlocksInfo(BlocksInfo): case _: assert False, f'{d}: invalid stats data' - varname,data = await res - Msg_r(', "{}_data": {}'.format( varname, json.dumps(dict(gen(data)),cls=json_encoder) )) + varname, data = await res + Msg_r(', "{}_data": {}'.format(varname, json.dumps(dict(gen(data)), cls=json_encoder))) - def process_stats_pre(self,i): pass + def process_stats_pre(self, i): pass def finalize_output(self): Msg('}') diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index c3d7c68..3329f07 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -14,66 +14,66 @@ mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes import asyncio from collections import namedtuple -from mmgen.util import msg,msg_r,is_int -from mmgen.term import get_term,get_terminal_size,get_char +from mmgen.util import msg, msg_r, is_int +from mmgen.term import get_term, get_terminal_size, get_char from mmgen.ui import line_input from .PollDisplay import PollDisplay -RED,RESET = ('\033[31m','\033[0m') -COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247,256)) + [231]] -ERASE_ALL,CUR_HOME = ('\033[J','\033[H') -CUR_HIDE,CUR_SHOW = ('\033[?25l','\033[?25h') +RED, RESET = ('\033[31m', '\033[0m') +COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247, 256)) + [231]] +ERASE_ALL, CUR_HOME = ('\033[J', '\033[H') +CUR_HIDE, CUR_SHOW = ('\033[?25l', '\033[?25h') term = None class Display(PollDisplay): poll_secs = 2 - def __init__(self,cfg): + def __init__(self, cfg): super().__init__(cfg) - global term,term_width + global term, term_width if not term: term = get_term() term.init(noecho=True) term_width = self.cfg.columns or get_terminal_size().width msg_r(CUR_HOME+ERASE_ALL+CUR_HOME) - async def get_info(self,rpc): + async def get_info(self, rpc): return await rpc.call('getpeerinfo') - def display(self,count): + def display(self, count): msg_r( CUR_HOME + (ERASE_ALL if count == 1 else '') + 'CONNECTED PEERS ({a}) {b} - poll {c}'.format( a = len(self.info), b = self.desc, - c = count ).ljust(term_width)[:term_width] + c = count).ljust(term_width)[:term_width] + '\n' + ('\n'.join(self.gen_display()) + '\n' if self.info else '') + ERASE_ALL + f"Type a peer number to disconnect, 'q' to quit, or any other key for {self.other_desc} display:" - + '\b' ) + + '\b') - async def disconnect_node(self,rpc,addr): - return await rpc.call('disconnectnode',addr) + async def disconnect_node(self, rpc, addr): + return await rpc.call('disconnectnode', addr) def get_input(self): - s = get_char(immed_chars='q0123456789',prehold_protect=False,num_bytes=1) + s = get_char(immed_chars='q0123456789', prehold_protect=False, num_bytes=1) if not is_int(s): return s with self.info_lock: msg('') term.reset() # readline required for correct operation here; without it, user must re-type first digit - ret = line_input( self.cfg, 'peer number> ', insert_txt=s, hold_protect=False ) + ret = line_input(self.cfg, 'peer number> ', insert_txt=s, hold_protect=False) term.init(noecho=True) self.enable_display = False # prevent display from updating before process_input() return ret - async def process_input(self,rpc): + async def process_input(self, rpc): ids = tuple(str(i['id']) for i in self.info) ret = False @@ -83,7 +83,7 @@ class Display(PollDisplay): from mmgen.exception import RPCFailure addr = self.info[ids.index(self.input)]['addr'] try: - await self.disconnect_node(rpc,addr) + await self.disconnect_node(rpc, addr) except RPCFailure: msg_r(f'Unable to disconnect peer {self.input} ({addr})') else: @@ -105,8 +105,8 @@ class BlocksDisplay(Display): def gen_display(self): - pd = namedtuple('peer_data',['id','blks_data','blks_width']) - bd = namedtuple('block_datum',['num','disp']) + pd = namedtuple('peer_data', ['id', 'blks_data', 'blks_width']) + bd = namedtuple('block_datum', ['num', 'disp']) def gen_block_data(): global min_height @@ -114,15 +114,15 @@ class BlocksDisplay(Display): for d in self.info: if d.get('inflight'): blocks = d['inflight'] - min_height = min(blocks) if not min_height else min(min_height,min(blocks)) - line = ' '.join(map(str,blocks))[:blks_field_width] + min_height = min(blocks) if not min_height else min(min_height, min(blocks)) + line = ' '.join(map(str, blocks))[:blks_field_width] blocks_disp = line.split() yield pd( d['id'], - [bd(blocks[i],blocks_disp[i]) for i in range(len(blocks_disp))], - len(line) ) + [bd(blocks[i], blocks_disp[i]) for i in range(len(blocks_disp))], + len(line)) else: - yield pd(d['id'],[],0) + yield pd(d['id'], [], 0) def gen_line(peer_data): for blk in peer_data.blks_data: @@ -136,7 +136,7 @@ class BlocksDisplay(Display): for peer_data in tuple(gen_block_data()): yield fs.format( peer_data.id, - ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width) ) + ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width)) class PeersDisplay(Display): @@ -152,5 +152,4 @@ class PeersDisplay(Display): A = id_width, b = d['addr'], B = addr_width, - c = d['subver'] - ).ljust(term_width)[:term_width] + c = d['subver']).ljust(term_width)[:term_width] diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 1de8a1a..8274e40 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -12,7 +12,7 @@ mmgen_node_tools.PollDisplay: update and display RPC data; get input from user """ -import sys,threading +import sys, threading from mmgen.util import msg from mmgen.term import get_char @@ -22,18 +22,18 @@ class PollDisplay: input = None poll_secs = 1 - def __init__(self,cfg): + def __init__(self, cfg): self.cfg = cfg self.info_lock = threading.Lock() # self.info accessed by 2 threads self.display_kill_flag = threading.Event() def get_input(self): - return get_char(immed_chars='q',prehold_protect=False,num_bytes=1) + return get_char(immed_chars='q', prehold_protect=False, num_bytes=1) - async def process_input(self,rpc): + async def process_input(self, rpc): return True - async def run(self,rpc): + async def run(self, rpc): async def do_display(): with self.info_lock: @@ -68,7 +68,7 @@ class PollDisplay: self.display_kill_flag.set() while True: - threading.Thread(target=get_input,daemon=True).start() + threading.Thread(target=get_input, daemon=True).start() await do_display() if await process_input(): break diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index df9f8ae..557b97a 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -19,28 +19,28 @@ mmgen_node_tools.Sound: audio-related functions for MMGen node tools """ -import sys,os,time +import sys, os, time from mmgen.util import die from mmgen_node_tools.Util import do_system _alsa_config_file = '/tmp/alsa-config-' + os.path.basename(sys.argv[0]) -_dvols = { 'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190 } +_dvols = {'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190} def timespec2secs(ts): import re - mul = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24 } + mul = {'s': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24} pat = r'^([0-9]+)([smhd]*)$' - m = re.match(pat,ts) + m = re.match(pat, ts) if m is None: die(2,"'%s': invalid time specifier" % ts) - a,b = m.groups() + a, b = m.groups() return int(a) * (mul[b] if b else 1) def parse_repeat_spec(rs): - return [(timespec2secs(i),timespec2secs(j)) - for i,j in [a.split(':') for a in rs.split(',')]] + return [(timespec2secs(i), timespec2secs(j)) + for i, j in [a.split(':') for a in rs.split(',')]] def init_sound(): def _restore_sound(): @@ -51,33 +51,33 @@ def init_sound(): atexit.register(_restore_sound) do_system('sudo alsactl store -f ' + _alsa_config_file) -def play_sound(fn,vol,repeat_spec='',remote_host='',kill_flg=None,testing=False): +def play_sound(fn, vol, repeat_spec='', remote_host='', kill_flg=None, testing=False): if not remote_host: do_system('sudo alsactl store -f ' + _alsa_config_file) - for k in 'Master','Speaker','Headphone': - do_system(('sudo amixer -q set %s on' % k),testing) + for k in 'Master', 'Speaker', 'Headphone': + do_system(('sudo amixer -q set %s on' % k), testing) # do_system('amixer -q set Headphone off') - vols = dict([(k,int(_dvols[k] * float(vol) / 100)) for k in _dvols]) + vols = dict([(k, int(_dvols[k] * float(vol) / 100)) for k in _dvols]) for k in vols: - do_system('sudo amixer -q set %s %s' % (k,vols[k]),testing) + do_system('sudo amixer -q set %s %s' % (k, vols[k]), testing) fn = os.path.expanduser(fn) cmd = ( 'aplay -q %s' % fn, - 'ssh %s mmnode-play-sound -v%d %s' % (remote_host,vol,fn) + 'ssh %s mmnode-play-sound -v%d %s' % (remote_host, vol, fn) )[bool(remote_host)] if repeat_spec and kill_flg: - for interval,duration in parse_repeat_spec(repeat_spec): + for interval, duration in parse_repeat_spec(repeat_spec): start = time.time() while time.time() < start + duration: - do_system(cmd,testing) + do_system(cmd, testing) if kill_flg.wait(interval): if not remote_host: do_system('sudo alsactl restore -f ' + _alsa_config_file) return else: # Play once - do_system(cmd,testing) + do_system(cmd, testing) if not remote_host: do_system('sudo alsactl restore -f ' + _alsa_config_file) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 95e03f2..ebefcf3 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -29,10 +29,10 @@ from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, lis from mmgen.ui import do_pager homedir = os.getenv('HOME') -dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') +dfl_cachedir = os.path.join(homedir, '.cache', 'mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' -asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) +asset_tuple = namedtuple('asset_tuple', ['symbol', 'id', 'source']) last_api_host = None percent_cols = { @@ -50,20 +50,19 @@ class DataSource: }, { 'fi': 'yahoospot', 'hi': 'yahoohist', - } - ] + }] @classmethod - def get_sources(cls,randomize=False): - g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups - return {k:v for a in g for k,v in a.items()} + def get_sources(cls, randomize=False): + g = random.sample(cls.source_groups, k=len(cls.source_groups)) if randomize else cls.source_groups + return {k: v for a in g for k, v in a.items()} class base: def fetch_delay(self): global last_api_host if not gcfg.testing and last_api_host and last_api_host != self.api_host: - delay = 1 + random.randrange(1,5000) / 1000 + delay = 1 + random.randrange(1, 5000) / 1000 msg_r(f'Waiting {delay:.3f} seconds...') time.sleep(delay) msg('') @@ -72,19 +71,18 @@ class DataSource: def get_data_from_network(self): curl_cmd = list_gen( - ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], + ['curl', '--tr-encoding', '--header', 'Accept: application/json', True], ['--compressed'], # adds 'Accept-Encoding: gzip' - ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], + ['--proxy', cfg.proxy, isinstance(cfg.proxy, str)], ['--silent', not cfg.verbose], - [self.api_url] - ) + [self.api_url]) if gcfg.testing: - Msg(fmt_list(curl_cmd,fmt='bare')) + Msg(fmt_list(curl_cmd, fmt='bare')) return try: - return run(curl_cmd,check=True,stdout=PIPE).stdout.decode() + return run(curl_cmd, check=True, stdout=PIPE).stdout.decode() except CalledProcessError as e: msg('') from .Misc import curl_exit_codes @@ -101,7 +99,7 @@ class DataSource: os.makedirs(cfg.cachedir) if not os.path.exists(self.json_fn): - open(self.json_fn,'w').write('{}') + open(self.json_fn, 'w').write('{}') use_cached_data = cfg.cached_data and not gcfg.download @@ -123,7 +121,7 @@ class DataSource: if gcfg.testing: return {} else: - die(1,self.rate_limit_errmsg(elapsed)) + die(1, self.rate_limit_errmsg(elapsed)) match data_type: case 'json': @@ -131,7 +129,7 @@ class DataSource: data = json.loads(data_in) except: self.json_data_error_msg(data_in) - die(2,'Retrieved data is not valid JSON, exiting') + die(2, 'Retrieved data is not valid JSON, exiting') json_text = data_in case 'python': data = data_in @@ -143,9 +141,9 @@ class DataSource: f'No cached {self.data_desc}! Run command without the --cached-data option, ' 'or use --download to retrieve data from remote host') else: - die(2,'Remote host returned no data!') + die(2, 'Remote host returned no data!') elif 'error' in data: - die(1,data['error']) + die(1, data['error']) if use_cached_data: if not cfg.quiet: @@ -162,15 +160,15 @@ class DataSource: return self.postprocess_data(data) - def json_data_error_msg(self,json_text): + def json_data_error_msg(self, json_text): pass - def postprocess_data(self,data): + def postprocess_data(self, data): return data @property def json_fn_rel(self): - return os.path.relpath(self.json_fn,start=homedir) + return os.path.relpath(self.json_fn, start=homedir) class coinpaprika(base): desc = 'CoinPaprika' @@ -185,30 +183,29 @@ class DataSource: def __init__(self): self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) - def rate_limit_errmsg(self,elapsed): + def rate_limit_errmsg(self, elapsed): return ( f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + - ('' if cfg.btc_only else ', or use --cached-data or --btc') - ) + ('' if cfg.btc_only else ', or use --cached-data or --btc')) @property def api_url(self): return ( f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else - f'https://{self.api_host}/v1/tickers' ) + f'https://{self.api_host}/v1/tickers') @property def json_fn(self): return os.path.join( cfg.cachedir, - 'ticker-btc.json' if cfg.btc_only else 'ticker.json' ) + 'ticker-btc.json' if cfg.btc_only else 'ticker.json') @property def timeout(self): return 0 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit - def json_data_error_msg(self,json_text): + def json_data_error_msg(self, json_text): tor_captcha_msg = f""" If you’re using Tor, the API request may have failed due to Captcha protection. A workaround for this issue is to retrieve the JSON data with a browser from @@ -223,20 +220,20 @@ class DataSource: Then invoke the program with --cached-data and without --btc """ msg(json_text[:1024] + '...') - msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) + msg(orange(fmt(tor_captcha_msg, strip_char='\t'))) - def postprocess_data(self,data): + def postprocess_data(self, data): return [data] if cfg.btc_only else data @staticmethod - def parse_asset_id(s,require_label): - sym,label = (*s.split('-',1),None)[:2] + def parse_asset_id(s, require_label): + sym, label = (*s.split('-', 1), None)[:2] if require_label and not label: - die(1,f'{s!r}: asset label is missing') + die(1, f'{s!r}: asset label is missing') return asset_tuple( symbol = sym.upper(), id = (s.lower() if label else None), - source = 'cc' ) + source = 'cc') class yahoospot(base): @@ -250,12 +247,12 @@ class DataSource: json_fn_basename = 'ticker-finance.json' @staticmethod - def get_id(sym,data): + def get_id(sym, data): return sym.lower() @staticmethod - def conv_data(sym,data,btcusd): - price_usd = Decimal( data['regularMarketPrice']['raw'] ) + def conv_data(sym, data, btcusd): + price_usd = Decimal(data['regularMarketPrice']['raw']) return { 'id': sym, 'name': data['shortName'], @@ -266,15 +263,14 @@ class DataSource: 'percent_change_30d': data['pct_chg_4wks'], 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, - 'last_updated': data['regularMarketTime'], - } + 'last_updated': data['regularMarketTime']} - def rate_limit_errmsg(self,elapsed): + def rate_limit_errmsg(self, elapsed): return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' @property def json_fn(self): - return os.path.join( cfg.cachedir, self.json_fn_basename ) + return os.path.join(cfg.cachedir, self.json_fn_basename) @property def timeout(self): @@ -282,37 +278,36 @@ class DataSource: @property def symbols(self): - return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + return [r.symbol for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi'] def get_data_from_network(self): kwargs = { 'formatted': True, 'asynchronous': True, - 'proxies': { 'https': cfg.proxy2 }, - } + 'proxies': {'https': cfg.proxy2}} if gcfg.test_suite: - kwargs.update({ 'timeout': 1, 'retry': 0 }) + kwargs.update({'timeout': 1, 'retry': 0}) if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( self.symbols, - fmt_dict(kwargs,fmt='kwargs') )) + fmt_dict(kwargs, fmt='kwargs'))) return from yahooquery import Ticker - return self.process_network_data( Ticker(self.symbols,**kwargs) ) + return self.process_network_data(Ticker(self.symbols,**kwargs)) - def process_network_data(self,ticker): + def process_network_data(self, ticker): return ticker.price @staticmethod - def parse_asset_id(s,require_label): + def parse_asset_id(s, require_label): return asset_tuple( symbol = s.upper(), id = s.lower(), - source = 'fi' ) + source = 'fi') class yahoohist(yahoospot): @@ -322,17 +317,17 @@ class DataSource: period = '1y' interval = '1wk' - def process_network_data(self,ticker): + def process_network_data(self, ticker): return ticker.history( period = self.period, interval = self.interval).to_json(orient='index') - def postprocess_data(self,data): + def postprocess_data(self, data): def gen(): keys = set() d = {} for key, val in data.items(): - if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): + if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$", key): date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) if (sym := m[1]) in keys: d[date] = val @@ -343,12 +338,12 @@ class DataSource: return dict(gen()) def assets_list_gen(cfg_in): - for k,v in cfg_in.cfg['assets'].items(): + for k, v in cfg_in.cfg['assets'].items(): yield '' yield k.upper() for e in v: - out = e.split('-',1) - yield ' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '') + out = e.split('-', 1) + yield ' {:5s} {}'.format(out[0], out[1] if len(out) == 2 else '') def gen_data(data): """ @@ -367,10 +362,9 @@ def gen_data(data): f'The symbol {dup_sym!r} is shared by the following assets:\n' + '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + - f'instead of {dup_sym!r}' - ) + f'instead of {dup_sym!r}') - def check_assets_found(wants,found,keys=['symbol','id']): + def check_assets_found(wants, found, keys=['symbol', 'id']): error = False for k in keys: missing = wants[k] - found[k] @@ -378,36 +372,32 @@ def gen_data(data): msg( ('The following IDs were not found in source data:\n{}' if k == 'id' else 'The following symbols could not be resolved:\n{}').format( - fmt_list(missing,fmt='col',indent=' ') - )) + fmt_list(missing, fmt='col', indent=' '))) error = True if error: - die(1,'Missing data, exiting') + die(1, 'Missing data, exiting') rows_want = { - 'id': {r.id for r in cfg.rows if isinstance(r,tuple) and r.id} - {'usd-us-dollar'}, - 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, - } + 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) usr_rate_assets_want = { 'id': {a.id for a in usr_rate_assets if a.id}, - 'symbol': {a.symbol for a in usr_rate_assets if not a.id} - } + 'symbol': {a.symbol for a in usr_rate_assets if not a.id}} usr_assets = cfg.usr_rows + cfg.usr_columns + tuple(c for c in (cfg.query or ()) if c) usr_wants = { 'id': ( {a.id for a in usr_assets + usr_rate_assets if a.id} - - {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'} ) + {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'}) , 'symbol': ( {a.symbol for a in usr_assets + usr_rate_assets if not a.id} - - {a.symbol for a in usr_assets if a.rate} - {'USD'} ), - } + {a.symbol for a in usr_assets if a.rate} - {'USD'})} - found = { 'id': set(), 'symbol': set() } + found = {'id': set(), 'symbol': set()} rate_assets = {} - wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} + wants = {k: rows_want[k] | usr_wants[k] for k in ('id', 'symbol')} for d in data['cc']: if d['id'] == 'btc-bitcoin': @@ -417,38 +407,37 @@ def gen_data(data): get_id = src_cls['fi'].get_id conv_func = src_cls['fi'].conv_data - for k,v in data['fi'].items(): - id = get_id(k,v) + for k, v in data['fi'].items(): + id = get_id(k, v) if wants['id']: if id in wants['id']: - if not isinstance(v,dict): + if not isinstance(v, dict): die(2, str(v)) if id in found['id']: - die(1,dup_sym_errmsg(id)) + die(1, dup_sym_errmsg(id)) if m := data['hi'].get(k): spot = v['regularMarketPrice']['raw'] hist = tuple(m.values()) v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( (spot / hist[-2]['close'] - 1) * 100, (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month - (spot / hist[0]['close'] - 1) * 100, - ) + (spot / hist[0]['close'] - 1) * 100) else: v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None - yield ( id, conv_func(id,v,btcusd) ) + yield (id, conv_func(id, v, btcusd)) found['id'].add(id) wants['id'].remove(id) if id in usr_rate_assets_want['id']: - rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key + rate_assets[k] = conv_func(id, v, btcusd) # NB: using symbol instead of ID for key else: break - for k in ('id','symbol'): + for k in ('id', 'symbol'): for d in data['cc']: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: - die(1,dup_sym_errmsg(d[k])) + die(1, dup_sym_errmsg(d[k])) if not 'price_usd' in d: d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd @@ -457,8 +446,9 @@ def gen_data(data): d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] # .replace('Z','+00:00') -- Python 3.9 backport - d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated'].replace('Z','+00:00')).timestamp()) - yield (d['id'],d) + d['last_updated'] = int(datetime.datetime.fromisoformat( + d['last_updated'].replace('Z', '+00:00')).timestamp()) + yield (d['id'], d) found[k].add(d[k]) wants[k].remove(d[k]) if d[k] in usr_rate_assets_want[k]: @@ -466,7 +456,7 @@ def gen_data(data): else: break - check_assets_found(usr_wants,found) + check_assets_found(usr_wants, found) for asset in (cfg.usr_rows + cfg.usr_columns): if asset.rate: @@ -475,14 +465,13 @@ def gen_data(data): """ _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() ra_rate = rate_assets[asset.rate_asset.symbol]['price_usd'] if asset.rate_asset else 1 - yield ( _id, { + yield (_id, { 'symbol': asset.symbol, 'id': _id, 'name': ' '.join(_id.split('-')[1:]), 'price_usd': ra_rate / asset.rate, 'price_btc': ra_rate / asset.rate / btcusd, - 'last_updated': None, - }) + 'last_updated': None}) yield ('usd-us-dollar', { 'symbol': 'USD', @@ -490,21 +479,20 @@ def gen_data(data): 'name': 'US Dollar', 'price_usd': Decimal(1), 'price_btc': Decimal(1) / btcusd, - 'last_updated': None, - }) + 'last_updated': None}) def main(): def update_sample_file(usr_cfg_file): - usr_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() + usr_data = files('mmgen_node_tools').joinpath('data', os.path.basename(usr_cfg_file)).read_text() sample_file = usr_cfg_file + '.sample' sample_data = open(sample_file).read() if os.path.exists(sample_file) else None if usr_data != sample_data: - os.makedirs(os.path.dirname(sample_file),exist_ok=True) + os.makedirs(os.path.dirname(sample_file), exist_ok=True) msg('{} {}'.format( - ('Updating','Creating')[sample_data is None], - sample_file )) - open(sample_file,'w').write(usr_data) + ('Updating', 'Creating')[sample_data is None], + sample_file)) + open(sample_file, 'w').write(usr_data) try: from importlib.resources import files # Python 3.9 @@ -515,19 +503,19 @@ def main(): update_sample_file(cfg_in.portfolio_file) if gcfg.portfolio and not cfg_in.portfolio: - die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( - os.path.relpath(cfg_in.portfolio_file,start=homedir))) + die(1, 'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( + os.path.relpath(cfg_in.portfolio_file, start=homedir))) if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: if not gcfg.download in DataSource.get_sources(): - die(1,f'{gcfg.download!r}: invalid data source') + die(1, f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: src_ids = DataSource.get_sources(randomize=True) - src_data = { k: src_cls[k]().get_data() for k in src_ids } + src_data = {k: src_cls[k]().get_data() for k in src_ids} if gcfg.testing: return @@ -542,24 +530,24 @@ def main(): data = dict(gen_data(src_data)) (do_pager if cfg.pager else Msg_r)( - '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n') + '\n'.join(getattr(Ticker, cfg.clsname)(data).gen_output()) + '\n') def make_cfg(gcfg_arg): - query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) + query_tuple = namedtuple('query', ['asset', 'to_asset']) + asset_data = namedtuple('asset_data', ['symbol', 'id', 'amount', 'rate', 'rate_asset', 'source']) - def parse_asset_id(s,require_label=False): - return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) + def parse_asset_id(s, require_label=False): + return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) def get_rows_from_cfg(add_data=None): def gen(): - for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): + for n, (k, v) in enumerate(cfg_in.cfg['assets'].items()): yield k if add_data and k in add_data: v += tuple(add_data[k]) for e in v: - yield parse_asset_id(e,require_label=True) + yield parse_asset_id(e, require_label=True) return tuple(gen()) def parse_percent_cols(arg): @@ -568,17 +556,19 @@ def make_cfg(gcfg_arg): res = arg.lower().split(',') for s in res: if s not in percent_cols: - die(1,f'{arg!r}: invalid --percent-cols parameter (valid letters: {fmt_list(percent_cols)})') + die(1, '{!r}: invalid --percent-cols parameter (valid letters: {})'.format( + arg, + fmt_list(percent_cols))) return res - def parse_usr_asset_arg(key,use_cf_file=False): + def parse_usr_asset_arg(key, use_cf_file=False): """ asset_id[:rate[:rate_asset]] """ def parse_parm(s): ss = s.split(':') - assert len(ss) in (1,2,3), f'{s}: malformed argument' - asset_id,rate,rate_asset = (*ss,None,None)[:3] + assert len(ss) in (1, 2, 3), f'{s}: malformed argument' + asset_id, rate, rate_asset = (*ss, None, None)[:3] parsed_id = parse_asset_id(asset_id) return asset_data( @@ -588,19 +578,19 @@ def make_cfg(gcfg_arg): rate = ( None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else - Decimal(rate) ), + Decimal(rate)), rate_asset = parse_asset_id(rate_asset) if rate_asset else None, - source = parsed_id.source ) + source = parsed_id.source) - cl_opt = getattr(gcfg,key) + cl_opt = getattr(gcfg, key) cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] - return tuple( parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt) ) + return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) def parse_query_arg(s): """ asset_id:amount[:to_asset_id[:to_amount]] """ - def parse_query_asset(asset_id,amount): + def parse_query_asset(asset_id, amount): parsed_id = parse_asset_id(asset_id) return asset_data( symbol = parsed_id.symbol, @@ -608,21 +598,20 @@ def make_cfg(gcfg_arg): amount = None if amount is None else Decimal(amount), rate = None, rate_asset = None, - source = parsed_id.source ) + source = parsed_id.source) ss = s.split(':') - assert len(ss) in (2,3,4), f'{s}: malformed argument' - asset_id,amount,to_asset_id,to_amount = (*ss,None,None)[:4] + assert len(ss) in (2, 3, 4), f'{s}: malformed argument' + asset_id, amount, to_asset_id, to_amount = (*ss, None, None)[:4] return query_tuple( - asset = parse_query_asset(asset_id,amount), - to_asset = parse_query_asset(to_asset_id,to_amount) if to_asset_id else None - ) + asset = parse_query_asset(asset_id, amount), + to_asset = parse_query_asset(to_asset_id, to_amount) if to_asset_id else None) - def gen_uniq(obj_list,key,preload=None): - found = set([getattr(obj,key) for obj in preload if hasattr(obj,key)] if preload else ()) + def gen_uniq(obj_list, key, preload=None): + found = set([getattr(obj, key) for obj in preload if hasattr(obj, key)] if preload else ()) for obj in obj_list: - id = getattr(obj,key) + id = getattr(obj, key) if id not in found: yield obj found.add(id) @@ -632,39 +621,38 @@ def make_cfg(gcfg_arg): 'user_added', usr_rows + (tuple(asset for asset in query if asset) if query else ()) + - usr_columns ) + usr_columns) def get_portfolio_assets(ret=()): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e,require_label=True) for e in cfg_in.portfolio) - return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') ) + ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) + return ('portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC')) def get_portfolio(): - return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'} + return {k: Decimal(v) for k, v in cfg_in.portfolio.items() + if (not gcfg.btc) or k == 'btc-bitcoin'} def parse_add_precision(arg): if not arg: return 0 s = str(arg) if not (s.isdigit() and s.isascii()): - die(1,f'{s}: invalid parameter for --add-precision (not an integer)') + die(1, f'{s}: invalid parameter for --add-precision (not an integer)') if int(s) > 30: - die(1,f'{s}: invalid parameter for --add-precision (value >30)') + die(1, f'{s}: invalid parameter for --add-precision (value >30)') return int(s) def create_rows(): rows = ( ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin',parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None ) - ) + ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else + get_rows_from_cfg(add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None)) - for hdr,data in ( - (get_usr_assets(),) if query else - (get_usr_assets(), get_portfolio_assets()) - ): + for hdr, data in ( + (get_usr_assets(),) if query else + (get_usr_assets(), get_portfolio_assets())): if data: - uniq_data = tuple(gen_uniq(data,'symbol',preload=rows)) + uniq_data = tuple(gen_uniq(data, 'symbol', preload=rows)) if uniq_data: rows += (hdr,) + uniq_data return rows @@ -693,26 +681,25 @@ def make_cfg(gcfg_arg): 'quiet', 'verbose']) - global gcfg,cfg_in,src_cls,cfg + global gcfg, cfg_in, src_cls, cfg gcfg = gcfg_arg - src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() } + src_cls = {k: getattr(DataSource, v) for k, v in DataSource.get_sources().items()} fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args cfg_in = get_cfg_in() usr_rows = parse_usr_asset_arg('add_rows') - usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) + usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None def get_proxy(name): - proxy = getattr(gcfg,name) + proxy = getattr(gcfg, name) return ( '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') - else (proxy or cfg_in.cfg.get(name)) - ) + else (proxy or cfg_in.cfg.get(name))) proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy @@ -723,7 +710,7 @@ def make_cfg(gcfg_arg): usr_rows = usr_rows, usr_columns = usr_columns, query = query, - adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ), + adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', btc_only = gcfg.btc or cfg_in.cfg.get('btc'), add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), @@ -745,17 +732,16 @@ def make_cfg(gcfg_arg): thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), - verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), - ) + verbose = gcfg.verbose or cfg_in.cfg.get('verbose')) def get_cfg_in(): - ret = namedtuple('cfg_in_data',['cfg','portfolio','cfg_file','portfolio_file']) - cfg_file,portfolio_file = ( - [os.path.join(gcfg.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)] - ) - cfg_data,portfolio_data = ( - [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None for fn in (cfg_file,portfolio_file)] - ) + ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) + cfg_file, portfolio_file = ( + [os.path.join(gcfg.data_dir_root, 'node_tools', fn) + for fn in (cfg_fn, portfolio_fn)]) + cfg_data, portfolio_data = ( + [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None + for fn in (cfg_file, portfolio_file)]) return ret( cfg = cfg_data or { 'assets': { @@ -765,14 +751,11 @@ def get_cfg_in(): # Pound Sterling, Euro, Swiss Franc 'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ], # Dow Jones Industrials, Nasdaq 100, S&P 500 - 'index': [ '^dji', '^ixic', '^gspc' ], - }, - 'proxy': 'http://vpn-gw:8118' - }, - portfolio = portfolio_data, - cfg_file = cfg_file, - portfolio_file = portfolio_file, - ) + 'index': [ '^dji', '^ixic', '^gspc' ]}, + 'proxy': 'http://vpn-gw:8118'}, + portfolio = portfolio_data, + cfg_file = cfg_file, + portfolio_file = portfolio_file) class Ticker: @@ -781,43 +764,43 @@ class Ticker: offer = None to_asset = None - def __init__(self,data): + def __init__(self, data): self.comma = ',' if cfg.thousands_comma else '' - self.col1_wid = max(len('TOTAL'),( + self.col1_wid = max(len('TOTAL'), ( max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else - max(len(d['symbol']) for d in data.values()) - )) + 1 + max(len(d['symbol']) for d in data.values()))) + 1 - self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows] - self.col_usd_prices = {k:self.data[k]['price_usd'] for k in self.col_ids} + self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row + for row in cfg.rows] + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} - self.prices = {row.id:self.get_row_prices(row.id) - for row in self.rows if isinstance(row,tuple) and row.id in data} + self.prices = {row.id: self.get_row_prices(row.id) + for row in self.rows if isinstance(row, tuple) and row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') - def format_last_update_col(self,cross_assets=()): + def format_last_update_col(self, cross_assets=()): if cfg.elapsed: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X', time.gmtime(t)) + fmt_func = lambda t, now: time.strftime('%F %X', time.gmtime(t)) d = self.data max_w = 0 if cross_assets: last_updated_x = [d[a.id]['last_updated'] for a in cross_assets] - min_t = min( (int(n) for n in last_updated_x if isinstance(n,int) ), default=None ) + min_t = min((int(n) for n in last_updated_x if isinstance(n, int)), default=None) else: min_t = None for row in self.rows: - if isinstance(row,tuple): + if isinstance(row, tuple): try: - t = int( d[row.id]['last_updated'] ) + t = int(d[row.id]['last_updated']) except TypeError as e: d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) except KeyError as e: @@ -825,7 +808,7 @@ class Ticker: pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( - (min(t,min_t) if min_t else t), + (min(t, min_t) if min_t else t), now = now) max_w = max(len(t_fmt), max_w) @@ -836,7 +819,7 @@ class Ticker: self.uprec = {k: max(0, v+4) + cfg.add_prec for k, v in exp} self.uwid = {k: 12 + max(0, abs(v)-6) + cfg.add_prec for k, v in exp} - def get_id(self,asset): + def get_id(self, asset): if asset.id: return asset.id else: @@ -844,11 +827,11 @@ class Ticker: if d['symbol'] == asset.symbol: return d['id'] - def create_label(self,id): + def create_label(self, id): return self.data[id]['name'].upper() def gen_output(self): - yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now))) + yield 'Current time: {} UTC'.format(time.strftime('%F %X', time.gmtime(now))) for asset in self.usr_col_assets: if asset.symbol != 'USD': @@ -858,17 +841,16 @@ class Ticker: self.create_label(asset.id), usdprice, self.comma, - max(2, 4-usdprice.adjusted()) ) + max(2, 4-usdprice.adjusted())) - if hasattr(self,'subhdr'): + if hasattr(self, 'subhdr'): yield self.subhdr if self.show_adj: yield ( ('Offered price differs from spot' if self.offer else 'Adjusting prices') + ' by ' - + yellow('{:+.2f}%'.format( (self.adjust-1) * 100 )) - ) + + yellow('{:+.2f}%'.format((self.adjust-1) * 100))) yield '' @@ -879,7 +861,7 @@ class Ticker: yield self.table_hdr for row in self.rows: - if isinstance(row,str): + if isinstance(row, str): yield ('-' * self.hl_wid) else: try: @@ -896,21 +878,20 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym,amt in cfg.portfolio.items(): + for sym, amt in cfg.portfolio.items(): try: - yield self.fmt_row(self.data[sym],amt=amt) + yield self.fmt_row(self.data[sym], amt=amt) except KeyError: yield gray(f'(no data for {sym})') yield '-' * self.hl_wid if not cfg.btc_only: yield self.fs_num.format( lbl = 'TOTAL', pc3='', pc4='', pc1='', pc2='', upd='', amt='', - **{ k.replace('-','_'): v for k,v in self.prices['total'].items() } - ) + **{k.replace('-', '_'): v for k, v in self.prices['total'].items()}) class overview(base): - def __init__(self,data): + def __init__(self, data): self.data = data self.adjust = cfg.adjust self.show_adj = self.adjust != 1 @@ -922,25 +903,26 @@ class Ticker: self.format_last_update_col() if cfg.portfolio: - self.prices['total'] = { col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] - for row in self.rows if isinstance(row,tuple) and row.id in cfg.portfolio and row.id in data) - for col_id in self.col_ids } + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + for row in self.rows + if isinstance(row, tuple) and row.id in cfg.portfolio and row.id in data) + for col_id in self.col_ids} self.init_prec() self.init_fs() - def get_row_prices(self,id): + def get_row_prices(self, id): if id in self.data: d = self.data[id] - return { k: ( + return {k: ( d['price_btc'] if k == 'btc-bitcoin' else d['price_usd'] / self.col_usd_prices[k] - ) * self.adjust for k in self.col_ids } + ) * self.adjust for k in self.col_ids} - def fmt_row(self,d,amt=None,amt_fmt=None): + def fmt_row(self, d, amt=None, amt_fmt=None): def fmt_pct(n): - return gray(' --') if n is None else (red,green)[n>=0](f'{n:+7.2f}') + return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] @@ -957,53 +939,45 @@ class Ticker: pc4 = fmt_pct(d.get('percent_change_30d')), upd = d.get('last_updated_fmt'), amt = amt_fmt, - **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() } - ) + **{k.replace('-', '_'): v * (1 if amt is None else amt) for k, v in p.items()}) def init_fs(self): - col_prec = {'usd-us-dollar':2+cfg.add_prec,'btc-bitcoin':8+cfg.add_prec } # | self.uprec # Python 3.9 - col_prec.update(self.uprec) - col_wid = {'usd-us-dollar':8+cfg.add_prec,'btc-bitcoin':12+cfg.add_prec } # """ - col_wid.update(self.uwid) + col_prec = {'usd-us-dollar': 2+cfg.add_prec, 'btc-bitcoin': 8+cfg.add_prec} | self.uprec max_row = max( - ( (k,v['btc-bitcoin']) for k,v in self.prices.items() ), - key = lambda a: a[1] - ) - widths = { k: len('{:{}.{}f}'.format( self.prices[max_row[0]][k], self.comma, col_prec[k] )) - for k in self.col_ids } + ((k, v['btc-bitcoin']) for k, v in self.prices.items()), + key = lambda a: a[1]) + widths = {k: len('{:{}.{}f}'.format(self.prices[max_row[0]][k], self.comma, col_prec[k])) + for k in self.col_ids} - fd = namedtuple('format_str_data',['fs_str','fs_num','wid']) + fd = namedtuple('format_str_data', ['fs_str', 'fs_num', 'wid']) col_fs_data = { - 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid), + 'label': fd(f'{{lbl:{self.col1_wid}}}', f'{{lbl:{self.col1_wid}}}', self.col1_wid), 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), - 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2), - 'amt': fd(' {amt}', ' {amt}', 21), - } -# } | { k: fd( # Python 3.9 - col_fs_data.update({ k: fd( - ' {{{}:>{}}}'.format( k.replace('-','_'), widths[k] ), - ' {{{}:{}{}.{}f}}'.format( k.replace('-','_'), widths[k], self.comma, col_prec[k] ), - widths[k]+2 - ) for k in self.col_ids - }) + 'update_time': fd(' {upd}', ' {upd}', + max((19 if cfg.portfolio else 0), self.upd_w) + 2), + 'amt': fd(' {amt}', ' {amt}', 21) + } | {k: fd( + ' {{{}:>{}}}'.format(k.replace('-', '_'), widths[k]), + ' {{{}:{}{}.{}f}}'.format(k.replace('-', '_'), widths[k], self.comma, col_prec[k]), + widths[k] + 2 + ) for k in self.col_ids} cols = ( - ['label','usd-us-dollar'] + + ['label', 'usd-us-dollar'] + [asset.id for asset in self.usr_col_assets] + - [a for a,b in ( - ( 'btc-bitcoin', not cfg.btc_only ), - ( 'pct1y', 'y' in cfg.percent_cols ), - ( 'pct1m', 'm' in cfg.percent_cols ), - ( 'pct1w', 'w' in cfg.percent_cols ), - ( 'pct1d', 'd' in cfg.percent_cols ), - ( 'update_time', cfg.update_time ), - ) if b] - ) + [a for a, b in ( + ('btc-bitcoin', not cfg.btc_only), + ('pct1y', 'y' in cfg.percent_cols), + ('pct1m', 'm' in cfg.percent_cols), + ('pct1w', 'w' in cfg.percent_cols), + ('pct1d', 'd' in cfg.percent_cols), + ('update_time', cfg.update_time)) + if b]) cols2 = list(cols) if cfg.update_time: cols2.pop() @@ -1029,17 +1003,16 @@ class Ticker: amt = ' AMOUNT', usd_us_dollar = 'USD', btc_bitcoin = ' BTC', - **{ a.id.replace('-','_'): a.symbol for a in self.usr_col_assets } - ) + **{a.id.replace('-', '_'): a.symbol for a in self.usr_col_assets}) class trading(base): - def __init__(self,data): + def __init__(self, data): self.data = data self.asset = cfg.query.asset._replace(id=self.get_id(cfg.query.asset)) self.to_asset = ( cfg.query.to_asset._replace(id=self.get_id(cfg.query.to_asset)) - if cfg.query.to_asset else None ) + if cfg.query.to_asset else None) self.col_ids = [self.asset.id] self.adjust = cfg.adjust if self.to_asset: @@ -1048,12 +1021,13 @@ class Ticker: real_price = ( self.asset.amount * data[self.asset.id]['price_usd'] - / data[self.to_asset.id]['price_usd'] - ) + / data[self.to_asset.id]['price_usd']) if self.adjust != 1: - die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier') + die(1, + 'the --adjust option may not be combined with TO_AMOUNT ' + 'in the trade specifier') self.adjust = self.offer / real_price - self.hl_ids = [self.asset.id,self.to_asset.id] + self.hl_ids = [self.asset.id, self.to_asset.id] else: self.hl_ids = [self.asset.id] @@ -1070,10 +1044,10 @@ class Ticker: self.init_prec() self.init_fs() - def get_row_prices(self,id): + def get_row_prices(self, id): if id in self.data: d = self.data[id] - return { k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids } + return {k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids} def init_fs(self): self.max_wid = max( @@ -1081,10 +1055,8 @@ class Ticker: v[self.asset.id] * self.asset.amount, 16 + cfg.add_prec, self.comma, - 8 + cfg.add_prec - )) - for v in self.prices.values() - ) + 8 + cfg.add_prec)) + for v in self.prices.values()) self.fs_str = '{lbl:%s} {p_spot}' % self.col1_wid self.hl_wid = self.col1_wid + self.max_wid + 1 if self.show_adj: @@ -1094,20 +1066,19 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self,d): + def fmt_row(self, d): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount - p_spot = '{:{}{}.{}f}'.format( p, self.max_wid, self.comma, 8+cfg.add_prec ) + p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) p_adj = ( - '{:{}{}.{}f}'.format( p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec ) - if self.show_adj else '' ) + '{:{}{}.{}f}'.format(p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec) + if self.show_adj else '') return self.fs_str.format( lbl = self.create_label(id) if cfg.name_labels else d['symbol'], p_spot = green(p_spot) if id in self.hl_ids else p_spot, p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, - upd = d.get('last_updated_fmt'), - ) + upd = d.get('last_updated_fmt')) @property def table_hdr(self): @@ -1115,12 +1086,11 @@ class Ticker: lbl = '', p_spot = '{t:>{w}}'.format( t = 'SPOT PRICE', - w = self.max_wid ), + w = self.max_wid), p_adj = '{t:>{w}}'.format( t = ('OFFERED' if self.offer else 'ADJUSTED') + ' PRICE', - w = self.max_wid ), - upd = 'UPDATED' - ) + w = self.max_wid), + upd = 'UPDATED') @property def subhdr(self): @@ -1133,9 +1103,8 @@ class Ticker: ) + ( ( ' =>' + - (' {:{}}'.format(self.offer,self.comma) if self.offer else '') + + (' {:{}}'.format(self.offer, self.comma) if self.offer else '') + ' {} ({})'.format( self.to_asset.symbol, - self.create_label(self.to_asset.id) ) - ) if self.to_asset else '' ) - ) + self.create_label(self.to_asset.id)) + ) if self.to_asset else '')) diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 1d5426b..f36faad 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -21,53 +21,52 @@ mmgen_node_tools.Util: utility functions for MMGen node tools import time -def get_hms(t=None,utc=False,no_secs=False): +def get_hms(t=None, utc=False, no_secs=False): secs = t or time.time() - ret = (time.localtime,time.gmtime)[utc](secs) - fs,n = (('{:02}:{:02}:{:02}',6),('{:02}:{:02}',5))[no_secs] + ret = (time.localtime, time.gmtime)[utc](secs) + fs, n = (('{:02}:{:02}:{:02}', 6), ('{:02}:{:02}', 5))[no_secs] return fs.format(*ret[3:n]) -def get_day_hms(t=None,utc=False): +def get_day_hms(t=None, utc=False): secs = t or time.time() - ret = (time.localtime,time.gmtime)[utc](secs) + ret = (time.localtime, time.gmtime)[utc](secs) return '{:04}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*ret[0:6]) -def do_system(cmd,testing=False,shell=False): +def do_system(cmd, testing=False, shell=False): if testing: from mmgen.util import msg msg("Would execute: '%s'" % cmd) return True else: import subprocess - return subprocess.call((cmd if shell else cmd.split()),shell,stderr=subprocess.PIPE) + return subprocess.call((cmd if shell else cmd.split()), shell, stderr=subprocess.PIPE) -def get_url(url,gzip_ok=False,proxy=None,timeout=60,verbose=False,debug=False): +def get_url(url, gzip_ok=False, proxy=None, timeout=60, verbose=False, debug=False): if debug: print('get_url():') print(' url', url) - print(' gzip_ok:',gzip_ok, 'proxy:',proxy, 'timeout:',timeout, 'verbose:',verbose) - import pycurl,io + print(' gzip_ok:', gzip_ok, 'proxy:', proxy, 'timeout:', timeout, 'verbose:', verbose) + import pycurl, io c = pycurl.Curl() c_out = io.StringIO() - c.setopt(pycurl.WRITEFUNCTION,c_out.write) - c.setopt(pycurl.TIMEOUT,timeout) - c.setopt(pycurl.FOLLOWLOCATION,True) - c.setopt(pycurl.COOKIEFILE,'') - c.setopt(pycurl.VERBOSE,verbose) + c.setopt(pycurl.WRITEFUNCTION, c_out.write) + c.setopt(pycurl.TIMEOUT, timeout) + c.setopt(pycurl.FOLLOWLOCATION, True) + c.setopt(pycurl.COOKIEFILE, '') + c.setopt(pycurl.VERBOSE, verbose) if gzip_ok: - c.setopt(pycurl.USERAGENT,'Lynx/2.8.9dev.8 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.4.9') + c.setopt(pycurl.USERAGENT, 'Lynx/2.8.9dev.8 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.4.9') c.setopt(pycurl.HTTPHEADER, [ 'Accept: text/html, text/plain, text/sgml, text/css, application/xhtml+xml, */*;q=0.01', 'Accept-Encoding: gzip', - 'Accept-Language: en'] - ) + 'Accept-Language: en']) if proxy: - c.setopt(pycurl.PROXY,proxy) - c.setopt(pycurl.URL,url) + c.setopt(pycurl.PROXY, proxy) + c.setopt(pycurl.URL, url) c.perform() text = c_out.getvalue() if text[:2] == '\x1f\x8b': # gzip magic number - c_out.seek(0,0) + c_out.seek(0, 0) import gzip with gzip.GzipFile(fileobj=c_out) as f: text = f.read() @@ -103,20 +102,19 @@ big_digits = { """ } -_bnums_c,_bpunc_c = [[l.strip('\n') + ' ' * (big_digits[m]*big_digits['n']) +_bnums_c, _bpunc_c = [[l.strip('\n') + ' ' * (big_digits[m]*big_digits['n']) for l in big_digits[k][1:].split('\n')] - for k,m in (('nums','w'),('punc','pw'))] + for k, m in (('nums', 'w'), ('punc', 'pw'))] -_bnums_n,_bpunc_n = [[[l[0+(j*w):w+(j*w)] for l in i] - for j in range(big_digits[n])] for n,w,i in - (('n',big_digits['w'],_bnums_c),('pn',big_digits['pw'],_bpunc_c))] +_bnums_n, _bpunc_n = [[[l[0+(j*w):w+(j*w)] for l in i] + for j in range(big_digits[n])] for n, w, i in + (('n', big_digits['w'], _bnums_c), ('pn', big_digits['pw'], _bpunc_c))] -def display_big_digits(s,pre='',suf=''): - s = [int((d,10,11)[(d in '.:')+(d==':')]) for d in s] +def display_big_digits(s, pre='', suf=''): + s = [int((d, 10, 11)[(d in '.:')+(d==':')]) for d in s] return pre + ('\n'+pre).join( - [''.join([(_bnums_n+_bpunc_n)[d][l] for d in s]) + suf for l in range(big_digits['h'])] - ) + [''.join([(_bnums_n+_bpunc_n)[d][l] for d in s]) + suf for l in range(big_digits['h'])]) if __name__ == '__main__': num = '2345.17' - print(display_big_digits(num,pre='+ ',suf=' +')) + print(display_big_digits(num, pre='+ ', suf=' +')) diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 37f0482..4647029 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -16,7 +16,7 @@ import sys from mmgen.obj import CoinTxID from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,suf,make_timestr,async_run +from mmgen.util import msg, Msg, die, suf, make_timestr, async_run from mmgen.color import red opts_data = { @@ -32,16 +32,16 @@ opts_data = { } } -def do_output(proto,addr_data,blk_hdrs): +def do_output(proto, addr_data, blk_hdrs): col1w = len(str(len(addr_data))) indent = ' ' * (col1w + 2) - for n,(addr,unspents) in enumerate(addr_data.items(),1): + for n, (addr, unspents) in enumerate(addr_data.items(), 1): Msg(f'\n{n:{col1w}}) Address: {addr.hl(addr.view_pref)}') if unspents: - heights = { u['height'] for u in unspents } + heights = {u['height'] for u in unspents} Msg('{}Balance: {}'.format( indent, sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), @@ -50,22 +50,21 @@ def do_output(proto,addr_data,blk_hdrs): red(str(len(unspents))), suf(unspents), red(str(len(heights))), - suf(heights) )) + suf(heights))) blk_w = len(str(unspents[-1]['height'])) - fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent,max(5,blk_w)) - Msg(fs.format('Block','Date','TxID','Vout',' Amount')) + fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent, max(5, blk_w)) + Msg(fs.format('Block', 'Date', 'TxID', 'Vout', ' Amount')) for u in unspents: Msg(fs.format( u['height'], - make_timestr( blk_hdrs[u['height']]['time'] ), + make_timestr(blk_hdrs[u['height']]['time']), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(6, color=True, prec=8) - )) + proto.coin_amt(u['amount']).fmt(6, color=True, prec=8))) else: Msg(f'{indent}No balance') -def do_output_tabular(proto,addr_data,blk_hdrs): +def do_output_tabular(proto, addr_data, blk_hdrs): col1w = len(str(len(addr_data))) + 1 max_addrw = max(len(addr) for addr in addr_data) @@ -75,9 +74,9 @@ def do_output_tabular(proto,addr_data,blk_hdrs): lb_w = max(len(h) for h in lb_heights) fs = ( - ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w,max(5,fb_w),max(4,lb_w)) + ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w, max(5, fb_w), max(4, lb_w)) if cfg.first_block else - ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w,max(4,lb_w)) ) + ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w, max(4, lb_w))) Msg('\n' + fs.format( n = '', @@ -87,20 +86,19 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = 'Block', B = 'Last', T = 'Block', - A = ' Amount' )) + A = ' Amount')) - for n,(addr,unspents) in enumerate(addr_data.items(),1): + for n, (addr, unspents) in enumerate(addr_data.items(), 1): if unspents: Msg(fs.format( n = str(n) + ')', a = addr.fmt(addr.view_pref, max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], - t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), + t = make_timestr(blk_hdrs[unspents[0]['height']]['time']), B = unspents[-1]['height'], - T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8) - )) + T = make_timestr(blk_hdrs[unspents[-1]['height']]['time']), + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8))) else: Msg(fs.format( n = str(n) + ')', @@ -110,61 +108,61 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = '', B = '-', T = '', - A = ' -' )) + A = ' -')) async def main(req_addrs): proto = cfg._proto from mmgen.addr import CoinAddr - addrs = [CoinAddr(proto,addr) for addr in req_addrs] + addrs = [CoinAddr(proto, addr) for addr in req_addrs] from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,ignore_wallet=True) + rpc = await rpc_init(cfg, ignore_wallet=True) height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') from mmgen.proto.btc.misc import scantxoutset - res = await scantxoutset( cfg, rpc, [f'addr({addr})' for addr in addrs] ) + res = await scantxoutset(cfg, rpc, [f'addr({addr})' for addr in addrs]) if not res['success']: - die(1,'UTXO scanning failed or was interrupted') + die(1, 'UTXO scanning failed or was interrupted') elif not res['unspents']: msg('Address has no balance' if len(addrs) == 1 else - 'Addresses have no balances' ) + 'Addresses have no balances') else: addr_data = {k:[] for k in addrs} if 'desc' in res['unspents'][0]: import re - for unspent in sorted(res['unspents'],key=lambda x: x['height']): - addr = re.match('addr\((.*?)\)',unspent['desc'])[1] + for unspent in sorted(res['unspents'], key=lambda x: x['height']): + addr = re.match('addr\((.*?)\)', unspent['desc'])[1] addr_data[addr].append(unspent) else: from mmgen.proto.btc.tx.base import decodeScriptPubKey - for unspent in sorted(res['unspents'],key=lambda x: x['height']): + for unspent in sorted(res['unspents'], key=lambda x: x['height']): ds = decodeScriptPubKey(proto, unspent['scriptPubKey']) addr_data[ds.addr].append(unspent) good_addrs = len([v for v in addr_data.values() if v]) Msg('Total: {} in {} address{}'.format( - proto.coin_amt(res['total_amount']).hl2(unit=True,fs='{:,}'), + proto.coin_amt(res['total_amount']).hl2(unit=True, fs='{:,}'), red(str(good_addrs)), - suf(good_addrs,'es') - )) + suf(good_addrs, 'es'))) blk_heights = {i['height'] for i in res['unspents']} blk_hashes = await rpc.batch_call('getblockhash', [(h,) for h in blk_heights]) blk_hdrs = await rpc.batch_call('getblockheader', [(H,) for H in blk_hashes]) - (do_output_tabular if cfg.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) + (do_output_tabular if cfg.tabular else do_output)( + proto, addr_data, dict(zip(blk_heights, blk_hdrs))) -cfg = Config( opts_data=opts_data, init_opts={'rpc_backend':'aiohttp'} ) +cfg = Config(opts_data=opts_data, init_opts={'rpc_backend': 'aiohttp'}) if len(cfg._args) < 1: - die(1,'This command requires at least one coin address argument') + die(1, 'This command requires at least one coin address argument') try: async_run(cfg, main, args=[cfg._args]) diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 18136c4..eaed15b 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -20,9 +20,9 @@ mmnode-blocks-info: Display information about a block or range of blocks """ -from mmgen.cfg import gc,Config -from mmgen.util import async_run,fmt_list -from .BlocksInfo import BlocksInfo,JSONBlocksInfo +from mmgen.cfg import gc, Config +from mmgen.util import async_run, fmt_list +from .BlocksInfo import BlocksInfo, JSONBlocksInfo opts_data = { 'sets': [ @@ -145,14 +145,13 @@ EXAMPLES: $ {p} --rpc-backend=aio -H +1000 This program requires a txindex-enabled daemon for correct operation. -""" }, +"""}, 'code': { - 'notes': lambda cfg,proto,s: s.format( + 'notes': lambda cfg, proto, s: s.format( I = proto.diff_adjust_interval, - F = fmt_list(BlocksInfo.fields,fmt='bare'), - S = fmt_list(BlocksInfo.all_stats,fmt='bare'), - p = gc.prog_name, - ) + F = fmt_list(BlocksInfo.fields, fmt='bare'), + S = fmt_list(BlocksInfo.all_stats, fmt='bare'), + p = gc.prog_name) } } @@ -172,7 +171,7 @@ async def main(): await m.process_blocks() if m.last: - for i,sname in enumerate(m.stats): + for i, sname in enumerate(m.stats): m.process_stats_pre(i) await m.process_stats(sname) diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 3211f7f..188f60e 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -21,10 +21,10 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool """ from mmgen.cfg import Config -from mmgen.util import async_run,die,fmt,make_timestr,check_int_between -from mmgen.util2 import int2bytespec,parse_bytespec +from mmgen.util import async_run, die, fmt, make_timestr, check_int_between +from mmgen.util2 import int2bytespec, parse_bytespec -min_prec,max_prec,dfl_prec = (0,6,4) +min_prec, max_prec, dfl_prec = (0, 6, 4) fee_brackets = [ 1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, @@ -42,9 +42,9 @@ fee_brackets = [ opts_data = { 'sets': [ - ('detail',True,'ranges',True), - ('detail',True,'show_mb_col',True), - ('detail',True,'precision',6), + ('detail', True, 'ranges', True), + ('detail', True, 'show_mb_col', True), + ('detail', True, 'precision', 6), ], 'text': { 'desc': 'Visualize the fee structure of a node’s mempool', @@ -83,37 +83,37 @@ cfg = Config(opts_data=opts_data) if cfg.ignore_below: if cfg.show_empty: - die(1,'Conflicting options: --ignore-below, --show-empty') + die(1, 'Conflicting options: --ignore-below, --show-empty') ignore_below = parse_bytespec(cfg.ignore_below) precision = ( check_int_between(cfg.precision, min_prec, max_prec, desc='--precision arg') - if cfg.precision else dfl_prec ) + if cfg.precision else dfl_prec) from mmgen.term import get_terminal_size width = cfg.columns or get_terminal_size().width class fee_bracket: - def __init__(self,top,bottom): + def __init__(self, top, bottom): self.top = top self.bottom = bottom self.tx_bytes = 0 self.tx_bytes_cum = 0 self.skip = False -def log(data,fn): +def log(data, fn): import json from mmgen.rpc.util import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( cfg = cfg, outfile = fn, - data = json.dumps(data,cls=json_encoder,sort_keys=True,indent=4), + data = json.dumps(data, cls=json_encoder, sort_keys=True, indent=4), desc = 'mempool', - ask_overwrite = False ) + ask_overwrite = False) -def create_data(coin_amt,mempool): - out = [fee_bracket(fee_brackets[i],fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] +def create_data(coin_amt, mempool): + out = [fee_bracket(fee_brackets[i], fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] # populate fee brackets: size_key = 'size' if proto.coin == 'BCH' else 'vsize' @@ -143,7 +143,7 @@ def create_data(coin_amt,mempool): return out -def gen_header(host,mempool,blockcount): +def gen_header(host, mempool, blockcount): yield fmt(f""" Mempool Fee Structure @@ -159,27 +159,26 @@ def gen_header(host,mempool,blockcount): elif cfg.ignore_below: yield 'Ignoring fee brackets with less than {:,} bytes ({})'.format( ignore_below, - int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), - ) + int2bytespec(ignore_below, 'MB', '0.6', strip=True, add_space=True)) if cfg.include_current: yield 'Including transactions in current fee bracket in Total MB amounts' def fmt_mb(n): - return int2bytespec(n,'MB',f'0.{precision}',print_sym=False) + return int2bytespec(n, 'MB', f'0.{precision}', print_sym=False) def gen_body(data): - tx_bytes_max = max((i.tx_bytes for i in data),default=0) - top_max = max((i.top for i in data),default=0) - bot_max = max((i.bottom for i in data),default=0) - col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'),6) + tx_bytes_max = max((i.tx_bytes for i in data), default=0) + top_max = max((i.top for i in data), default=0) + bot_max = max((i.bottom for i in data), default=0) + col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'), 6) col2_w = len(fmt_mb(tx_bytes_max)) if cfg.show_mb_col else 0 col3_w = len(fmt_mb(data[-1].tx_bytes_cum)) if data else 0 col4_w = width - col1_w - col2_w - col3_w - (4 if col2_w else 3) if cfg.show_mb_col: - fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w,col2_w,col3_w) + fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w, col2_w, col3_w) else: - fs = '{a:<%i} {c:>%i} {d}' % (col1_w,col3_w) + fs = '{a:<%i} {c:>%i} {d}' % (col1_w, col3_w) yield fs.format(a='', b='', c=f'{"Total":<{col3_w}}', d='') yield fs.format(a='sat/B', b=f'{"MB":<{col2_w}}', c=f'{"MB":<{col3_w}}', d='') @@ -188,16 +187,16 @@ def gen_body(data): if not i.skip: cum_bytes = i.tx_bytes_cum + i.tx_bytes if cfg.include_current else i.tx_bytes_cum yield fs.format( - a = '{}-{}'.format(i.bottom,i.top) if cfg.ranges else i.top, + a = '{}-{}'.format(i.bottom, i.top) if cfg.ranges else i.top, b = fmt_mb(i.tx_bytes), c = fmt_mb(cum_bytes), - d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) ) + d = '-' * int(col4_w * (i.tx_bytes / tx_bytes_max))) yield fs.format( a = 'TOTAL', b = '', c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes if data else 0), - d = '' ) + d = '') async def main(): @@ -205,19 +204,19 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) - mempool = await c.call('getrawmempool',True) + mempool = await c.call('getrawmempool', True) if cfg.log: - log(mempool,'mempool.json') + log(mempool, 'mempool.json') - data = create_data(proto.coin_amt,mempool) + data = create_data(proto.coin_amt, mempool) cfg._util.stdout_or_pager( '\n'.join(gen_header( c.host, mempool, - await c.call('getblockcount') )) + '\n\n' + - '\n'.join(gen_body(data)) + '\n' ) + await c.call('getblockcount'))) + '\n\n' + + '\n'.join(gen_body(data)) + '\n') async_run(cfg, main) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 8f0a3a3..60c5440 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -28,7 +28,7 @@ from mmgen.util import async_run bdr_proj = 9.95 opts_data = { - 'sets': [('mined',True,'list',True)], + 'sets': [('mined', True, 'list', True)], 'text': { 'desc': 'Estimate date(s) of future block subsidy halving(s)', 'usage':'[opts]', @@ -41,7 +41,7 @@ opts_data = { {bdr_proj:.5f} min) -s, --sample-size=N Block range to calculate block discovery interval for next halving estimate (default: dynamically calculated) -""" } +"""} } cfg = Config(opts_data=opts_data) @@ -53,14 +53,14 @@ def date(t): return '{}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*time.gmtime(t)[:6]) def dhms(t): - t,neg = (-t,'-') if t < 0 else (t,' ') + t, neg = (-t, '-') if t < 0 else (t, ' ') return f'{neg}{t//60//60//24} days, {t//60//60%24:02}:{t//60%60:02}:{t%60:02} h/m/s' def time_diff_warning(t_diff): if abs(t_diff) > 60*60: print('Warning: block tip time is {} {} clock time!'.format( dhms(abs(t_diff)), - ('behind','ahead of')[t_diff<0])) + ('behind', 'ahead of')[t_diff<0])) async def main(): @@ -72,9 +72,9 @@ async def main(): tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' remaining = proto.halving_interval - tip % proto.halving_interval - sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1,max(remaining,144)) + sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1, max(remaining, 144)) - cur,old = await c.gathered_call('getblockstats',((tip,),(tip - sample_size,))) + cur, old = await c.gathered_call('getblockstats', ((tip,), (tip - sample_size,))) clock_time = int(time.time()) time_diff_warning(clock_time - cur['time']) @@ -98,8 +98,7 @@ async def main(): f'Current block discovery interval (over last {sample_size} blocks): {bdr/60:0.2f} min\n\n' f'Current clock time (UTC): {date(clock_time)}\n' f'Est. halving date (UTC): {date(t_next)}\n' - f'Est. time until halving: {dhms(cur["time"] + t_rem - clock_time)}' - ) + f'Est. time until halving: {dhms(cur["time"] + t_rem - clock_time)}') async def print_halvings(): halving_blocknums = [i*proto.halving_interval for i in range(proto.max_halvings+1)][1:] @@ -108,13 +107,13 @@ async def main(): nhist = len(hist_halvings) nSubsidy = int(proto.start_subsidy / proto.coin_amt.satoshi) - block0_hash = await c.call('getblockhash',0) - block0_date = (await c.call('getblock',block0_hash))['time'] + block0_hash = await c.call('getblockhash', 0) + block0_date = (await c.call('getblock', block0_hash))['time'] def gen_data(): total_mined = 0 date = block0_date - for n,blk in enumerate(halving_blocknums): + for n, blk in enumerate(halving_blocknums): mined = (nSubsidy >> n) * proto.halving_interval if n == 0: mined -= nSubsidy # subtract unspendable genesis block subsidy @@ -123,13 +122,11 @@ async def main(): bdi = ( (hist_halvings[n]['time'] - date) / (proto.halving_interval * 60) if n < nhist else bdr/60 if n == nhist - else bdr_proj - ) + else bdr_proj) date = ( hist_halvings[n]['time'] if n < nhist - else t_next + int((n - nhist) * halving_secs) - ) - yield ( n, sub, blk, mined, total_mined, bdi, date ) + else t_next + int((n - nhist) * halving_secs)) + yield (n, sub, blk, mined, total_mined, bdi, date) if sub == 0: break @@ -150,8 +147,7 @@ async def main(): e = 'BDI (mins)', f = 'SUBSIDY ({proto.coin})', g = f'MINED ({proto.coin})', - h = f'TOTAL MINED ({proto.coin})' - ) + h = f'TOTAL MINED ({proto.coin})') + '\n' + fs.format( a = '-' * 7, @@ -159,22 +155,20 @@ async def main(): c = '-' * 19, d = '-' * 2, e = '-' * 10, - f = '-' * 13, + f = '-' * 17, g = '-' * 17, - h = '-' * 17 - ) + h = '-' * 17) + '\n' + '\n'.join(fs.format( - a = n + 1, - b = blk, - c = date(t), - d = ' P' if n > nhist else '' if n < nhist else ' E', - e = f'{bdr:8.5f}', - f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), - g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), - h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) - ) for n, sub, blk, mined, total_mined, bdr, t in gen_data()) - ) + a = n + 1, + b = blk, + c = date(t), + d = ' P' if n > nhist else '' if n < nhist else ' E', + e = f'{bdr:8.5f}', + f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), + g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), + h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) + ) for n, sub, blk, mined, total_mined, bdr, t in gen_data())) if cfg.list: await print_halvings() diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index b273ac6..f5cf992 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -20,7 +20,7 @@ mmnode-netrate: Bitcoin daemon network rate monitor """ -import sys,time +import sys, time from mmgen.cfg import Config from mmgen.util import async_run @@ -32,39 +32,38 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -""" - } +"""} } cfg = Config(opts_data=opts_data) -ERASE_LINE,CUR_UP = '\033[K','\033[1A' +ERASE_LINE, CUR_UP = '\033[K', '\033[1A' async def main(): from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) async def get_data(): d = await c.call('getnettotals') - return [float(e) for e in (d['totalbytesrecv'],d['totalbytessent'],d['timemillis'])] + return [float(e) for e in (d['totalbytesrecv'], d['totalbytessent'], d['timemillis'])] - rs,ss,ts = (None,None,None) + rs, ss, ts = (None, None, None) while True: - r,s,t = await get_data() + r, s, t = await get_data() if rs is not None: sys.stderr.write( '\rrcvd: {:9.2f} kB/s\nsent: {:9.2f} kB/s '.format( (r-rs)/(t-ts), - (s-ss)/(t-ts) )) + (s-ss)/(t-ts))) time.sleep(2) if rs is not None: - sys.stderr.write('{}{}{}'.format(ERASE_LINE,CUR_UP,ERASE_LINE)) + sys.stderr.write('{}{}{}'.format(ERASE_LINE, CUR_UP, ERASE_LINE)) - rs,ss,ts = (r,s,t) + rs, ss, ts = (r, s, t) try: async_run(cfg, main) diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 35292ea..6a7cb38 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -27,8 +27,7 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -""" - } +"""} } from mmgen.cfg import Config @@ -37,9 +36,9 @@ cfg = Config(opts_data=opts_data) async def main(): from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,ignore_wallet=True) + rpc = await rpc_init(cfg, ignore_wallet=True) - from .PeerBlocks import BlocksDisplay,PeersDisplay + from .PeerBlocks import BlocksDisplay, PeersDisplay blocks = BlocksDisplay(cfg) peers = PeersDisplay(cfg) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 351cb1e..c36d886 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -208,29 +208,26 @@ Customize output by editing the file To add a portfolio, edit the file ~/{pf_cfg} -""" - }, +"""}, 'code': { 'options': lambda s: s.format( - dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), - ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), + dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), + ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, - pc = fmt_list(Ticker.percent_cols,fmt='bare'), - ), + pc = fmt_list(Ticker.percent_cols, fmt='bare')), 'notes': lambda s: s.format( - assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), - cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), - pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), + cfg = os.path.relpath(cfg_in.cfg_file, start=homedir), + pf_cfg = os.path.relpath(cfg_in.portfolio_file, start=homedir), al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), - fi = src_cls['fi'](), - ) + fi = src_cls['fi']()) } } import os -from mmgen.util import fmt_list,fmt_dict +from mmgen.util import fmt_list, fmt_dict from mmgen.cfg import Config from . import Ticker @@ -238,7 +235,7 @@ gcfg = Config(opts_data=opts_data, caller_post_init=True) Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, cfg_in, src_cls gcfg._post_init() diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 2411f36..0f9db5b 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -23,7 +23,7 @@ mmnode-txfind: Find a transaction in the blockchain or mempool import sys from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,is_hex_str,async_run +from mmgen.util import msg, Msg, die, is_hex_str, async_run opts_data = { 'text': { @@ -48,29 +48,26 @@ msg_data = { 'normal': { 'none': 'Transaction not found in blockchain or mempool', 'block': 'Transaction is in block {b} ({c} confirmations)', - 'mem': 'Transaction is in mempool', - }, + 'mem': 'Transaction is in mempool'}, 'quiet': { 'none': 'None', 'block': '{b} {c}', - 'mem': 'mempool', - } -} + 'mem': 'mempool'}} async def main(txid): if len(txid) != 64 or not is_hex_str(txid): - die(2,f'{txid}: invalid transaction ID') + die(2, f'{txid}: invalid transaction ID') if cfg.verbose: msg(f'TxID: {txid}') from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) exitval = 0 try: tip1 = await c.call('getblockcount') - ret = await c.call('getrawtransaction',txid,True) + ret = await c.call('getrawtransaction', txid, True) tip2 = await c.call('getblockcount') except: Msg('\r' + msgs['none']) @@ -90,6 +87,6 @@ cfg = Config(opts_data=opts_data) msgs = msg_data['quiet' if cfg.quiet else 'normal'] if len(cfg._args) != 1: - die(1,'One transaction ID must be specified') + die(1, 'One transaction ID must be specified') sys.exit(async_run(cfg, main, args=[cfg._args[0]])) diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index 4792dee..a504678 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -335,19 +335,19 @@ class CmdTestRegtest(CmdTestBase): us = await r.rpc_call('listunspent',wallet='miner') tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') - outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} + outputs = {p.addr: tx1_amt for p in pairs[:nTxs]} outputs.update({burn_addr: self.proto.coin_amt(tx_input['amount']) - (tx1_amt*nTxs) - fee}) return await do_tx( - [{ 'txid': tx_input['txid'], 'vout': 0 }], + [{'txid': tx_input['txid'], 'vout': 0}], outputs, await r.miner_wif) async def do_tx2(tx,pairno): fee = self.proto.coin_amt(fees[pairno], from_decimal=True) - outputs = {p.addr:tx2_amt for p in pairs} + outputs = {p.addr: tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) return await do_tx( - [{ 'txid': tx['txid'], 'vout': pairno }], + [{'txid': tx['txid'], 'vout': pairno}], outputs, pairs[pairno].wif ) diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py index 7c2fdf3..444ae08 100644 --- a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -294,15 +294,14 @@ class fake_data: 20 7303 7307 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 21 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 22 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 7398 7409 - """ - } + """} def make_data(): def gen_address_data(): for line in fake_data.addresses.strip().split('\n'): data = line.split(maxsplit=2) - yield (data[0], {k:v for k,v in zip(('id','addr','subver'),data)}) + yield (data[0], {k: v for k, v in zip(('id', 'addr', 'subver'), data)}) def gen_iterations_data(): for line in fake_data.iterations.strip().split('\n'): @@ -320,8 +319,7 @@ class fake_data: 'id': int(d['id']), 'addr': d['addr'], 'subver': d['subver'], - 'inflight': [int(n)+830000 for n in blocks[iter_no]], - } + 'inflight': [int(n)+830000 for n in blocks[iter_no]]} def gen_data(): for iter_no in iterations_data: From b9957f918298329ab9057a94946abccdaad9d038 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 09:57:02 +0000 Subject: [PATCH 56/81] mmnode-ticker: support `--http-timeout` option --- mmgen_node_tools/Ticker.py | 4 ++++ mmgen_node_tools/data/version | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index ebefcf3..caee658 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -75,6 +75,7 @@ class DataSource: ['--compressed'], # adds 'Accept-Encoding: gzip' ['--proxy', cfg.proxy, isinstance(cfg.proxy, str)], ['--silent', not cfg.verbose], + ['--connect-timeout', str(gcfg.http_timeout), gcfg.http_timeout], [self.api_url]) if gcfg.testing: @@ -290,6 +291,9 @@ class DataSource: if gcfg.test_suite: kwargs.update({'timeout': 1, 'retry': 0}) + if gcfg.http_timeout: + kwargs.update({'timeout': gcfg.http_timeout}) + if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( self.symbols, diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 9f32e9d..6f735af 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev1 +3.6.dev2 From 06340ed521de5ba0333e2d4bf20105cde60b0e8c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 12:44:54 +0000 Subject: [PATCH 57/81] restore Nix support --- mmgen_node_tools/data/version | 2 +- nix/curl-cffi.nix | 50 +++++++++++++++++++++++++++++++ nix/node-tools-packages.nix | 4 +-- nix/use-system-libs.patch | 23 ++++++++++++++ nix/yahooquery-noversioning.patch | 45 ---------------------------- nix/yahooquery.nix | 39 ++++++++++-------------- 6 files changed, 92 insertions(+), 71 deletions(-) create mode 100644 nix/curl-cffi.nix create mode 100644 nix/use-system-libs.patch delete mode 100644 nix/yahooquery-noversioning.patch diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6f735af..f9d163c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev2 +3.6.dev3 diff --git a/nix/curl-cffi.nix b/nix/curl-cffi.nix new file mode 100644 index 0000000..15f9764 --- /dev/null +++ b/nix/curl-cffi.nix @@ -0,0 +1,50 @@ +{ + stdenv, + lib, + python, + buildPythonPackage, + fetchPypi, + curl-impersonate-chrome, +}: + +buildPythonPackage rec { + pname = "curl-cffi"; + # version = "0.13.0"; # uses option PROXY_CREDENTIAL_NO_REUSE, unavailable in current libcurl + version = "0.10.0"; + pyproject = true; + + src = fetchPypi { + pname = "curl_cffi"; + version = version; + # hash = "sha256-YuzZCjgr1QI3UONgbgqnyxo6i6QcFCcLjl4Unr9yxco="; # 0.13.0 + hash = "sha256-PjezUmjKWEkvVO0CCuS1DDPuDeutQUXbn3RvBO1GbrA="; # 0.10.0 + }; + + patches = [ ./use-system-libs.patch ]; + + buildInputs = [ curl-impersonate-chrome ]; + + build-system = with python.pkgs; [ + cffi + setuptools + ]; + + dependencies = with python.pkgs; [ + cffi + certifi + typing-extensions + ]; + + env = lib.optionalAttrs stdenv.cc.isGNU { + NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types"; + }; + + pythonImportsCheck = [ "curl_cffi" ]; + + meta = with lib; { + description = "Python binding for curl-impersonate via cffi"; + homepage = "https://curl-cffi.readthedocs.io"; + license = licenses.mit; + maintainers = with maintainers; [ chuangzhu ]; + }; +} diff --git a/nix/node-tools-packages.nix b/nix/node-tools-packages.nix index 19792c7..1244425 100644 --- a/nix/node-tools-packages.nix +++ b/nix/node-tools-packages.nix @@ -6,7 +6,7 @@ }; python-packages = with python.pkgs; { - yahooquery = (pkgs.callPackage ./yahooquery.nix {}); # ticker - pyyaml = pyyaml; # ticker + yahooquery = (callPackage ./yahooquery.nix {}); # ticker + pyyaml = pyyaml; # ticker }; } diff --git a/nix/use-system-libs.patch b/nix/use-system-libs.patch new file mode 100644 index 0000000..1d04487 --- /dev/null +++ b/nix/use-system-libs.patch @@ -0,0 +1,23 @@ +diff --git a/scripts/build.py b/scripts/build.py +index b705a0d..9bfcaab 100644 +--- a/scripts/build.py ++++ b/scripts/build.py +@@ -105,7 +105,6 @@ def get_curl_libraries(): + ffibuilder = FFI() + system = platform.system() + root_dir = Path(__file__).parent.parent +-download_libcurl() + + + ffibuilder.set_source( +@@ -114,9 +113,7 @@ ffibuilder.set_source( + #include "shim.h" + """, + # FIXME from `curl-impersonate` +- libraries=get_curl_libraries(), +- extra_objects=get_curl_archives(), +- library_dirs=[arch["libdir"]], ++ libraries=["curl-impersonate-chrome"], + source_extension=".c", + include_dirs=[ + str(root_dir / "include"), diff --git a/nix/yahooquery-noversioning.patch b/nix/yahooquery-noversioning.patch deleted file mode 100644 index 4bd3297..0000000 --- a/nix/yahooquery-noversioning.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/pyproject.toml b/pyproject.toml -index 9d3fb29..399c215 100644 ---- a/pyproject.toml -+++ b/pyproject.toml -@@ -10,24 +10,24 @@ readme = "README.md" - - [tool.poetry.dependencies] - python = ">=3.8.1,<4.0" --requests = "^2.31.0" --pandas = "^2.0.3" --requests-futures = "^1.0.1" --tqdm = "^4.65.0" --lxml = "^4.9.3" --selenium = {version = "^4.10.0", optional = true} --beautifulsoup4 = "^4.12.2" -+requests = ">=2.31.0" -+pandas = ">=2.0.3" -+requests-futures = ">=1.0.1" -+tqdm = ">=4.65.0" -+lxml = ">=4.9.3" -+selenium = {version = ">=4.10.0", optional = true} -+beautifulsoup4 = ">=4.12.2" - - [tool.poetry.dev-dependencies] --pytest = "^7.4.0" --isort = "^5.0.0" --flake8 = "^6.0.0" --mypy = "^1.4.1" --pytest-cov = "^4.1.0" --black = "^23.7.0" --pre-commit = "^3.3.3" --ipython = "^8.0.0" --mkdocs-material = "^9.1.18" -+pytest = ">=7.4.0" -+isort = ">=5.0.0" -+flake8 = ">=6.0.0" -+mypy = ">=1.4.1" -+pytest-cov = ">=4.1.0" -+black = ">=23.7.0" -+pre-commit = ">=3.3.3" -+ipython = ">=8.0.0" -+mkdocs-material = ">=9.1.18" - - [build-system] - requires = ["poetry-core>=1.0.0"] diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix index 8b1535d..3e8925f 100644 --- a/nix/yahooquery.nix +++ b/nix/yahooquery.nix @@ -1,37 +1,30 @@ { lib, - pkgs, - fetchFromGitHub, + buildPythonPackage, + fetchPypi, + python, }: -with pkgs.python312.pkgs; - buildPythonPackage rec { pname = "yahooquery"; - version = "2.3.7"; + version = "2.4.1"; pyproject = true; - disabled = pythonOlder "3.8.1"; - - src = fetchFromGitHub { - owner = "dpguthrie"; - repo = "yahooquery"; - rev = "refs/tags/v${version}"; - hash = "sha256-Iyuni1SoTB6f7nNFhN5A8Gnv9kV78frjpqvvW8qd+/M="; + src = fetchPypi { + pname = "yahooquery"; + version = version; + hash = "sha256-GQPGXq5qEtlelFAGNHkhbAeEbwE7riojkXkTUxt/rls="; }; - patches = [ ./yahooquery-noversioning.patch ]; + build-system = with python.pkgs; [ hatchling ]; - build-system = [ poetry-core ]; - - dependencies = [ - requests # ^2.31.0 - pandas # ^2.0.3 - requests-futures # ^1.0.1 - tqdm # ^4.65.0 - lxml # ^4.9.3 - selenium # {version = ^4.10.0, optional = true} - beautifulsoup4 # ^4.12.2 + propagatedBuildInputs = with python.pkgs; [ + (callPackage ./curl-cffi.nix {}) # >=0.10.0 + pandas + requests-futures + tqdm + lxml + beautifulsoup4 ]; doCheck = false; # skip tests From de833d75afb3fa313f2541eb1a71819a9c1ae141 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:47 +0000 Subject: [PATCH 58/81] whitespace, variable renames --- mmgen_node_tools/Ticker.py | 6 ++++-- test/cmdtest_d/misc.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index caee658..d681a81 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -192,8 +192,10 @@ class DataSource: @property def api_url(self): return ( - f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else - f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else + f'https://{self.api_host}/v1/tickers/btc-bitcoin' + if cfg.btc_only else + f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' + if self.asset_limit else f'https://{self.api_host}/v1/tickers') @property diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9528b76..fb48282 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -106,10 +106,17 @@ class CmdTestScripts(CmdTestBase): shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' - def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): + def ticker( + self, + args = [], + expect_list = None, + cached_data = True, + exit_val = None): t = self.spawn( 'mmnode-ticker', - (['--cached-data'] if cached else []) + self.ticker_args + args, + (['--cached-data'] if cached_data else []) + + self.ticker_args + + args, exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) @@ -121,7 +128,7 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False) + t = self.ticker(cached_data=False) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') @@ -137,7 +144,6 @@ class CmdTestScripts(CmdTestBase): 'BTC 23250.77 1.00000000 ETH 1659.66 0.07138094' ]) - def ticker4(self): return self.ticker( ['--widest','--add-columns=eurusd=x,inr-indian-rupee:79.5'], From 313c7af4bbf165ed65e5acd56e2461663d54bec2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 59/81] mmgen-ticker: test caching JSON data --- MANIFEST.in | 1 + mmgen_node_tools/Ticker.py | 12 ++++++--- test/cmdtest_d/httpd/ticker.py | 27 +++++++++++++++++++ test/cmdtest_d/misc.py | 27 ++++++++++++++----- test/init.sh | 1 + .../fakemods/mmgen_node_tools/Ticker.py | 23 ++++++++++++++++ 6 files changed, 82 insertions(+), 9 deletions(-) create mode 100755 test/cmdtest_d/httpd/ticker.py create mode 100644 test/overlay/fakemods/mmgen_node_tools/Ticker.py diff --git a/MANIFEST.in b/MANIFEST.in index 77b53ac..2581100 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,5 +7,6 @@ include test/test-release.d/*.sh include test/modtest_d/*.py include test/cmdtest_d/*.py include test/cmdtest_d/include/cfg.py +include test/cmdtest_d/httpd/ticker.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index d681a81..bfe899b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -175,6 +175,7 @@ class DataSource: desc = 'CoinPaprika' data_desc = 'cryptocurrency data' api_host = 'api.coinpaprika.com' + api_proto = 'https' ratelimit = 240 btc_ratelimit = 10 net_data_type = 'json' @@ -192,11 +193,11 @@ class DataSource: @property def api_url(self): return ( - f'https://{self.api_host}/v1/tickers/btc-bitcoin' + f'{self.api_proto}://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else - f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' + f'{self.api_proto}://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else - f'https://{self.api_host}/v1/tickers') + f'{self.api_proto}://{self.api_host}/v1/tickers') @property def json_fn(self): @@ -697,6 +698,11 @@ def make_cfg(gcfg_arg): cmd_args = gcfg._args cfg_in = get_cfg_in() + if gcfg.test_suite: # required for testing with overlay + from . import Ticker as this_mod + this_mod.src_cls = src_cls + this_mod.cfg_in = cfg_in + usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None diff --git a/test/cmdtest_d/httpd/ticker.py b/test/cmdtest_d/httpd/ticker.py new file mode 100755 index 0000000..2d946ae --- /dev/null +++ b/test/cmdtest_d/httpd/ticker.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# MMGen Node Tools, terminal-based programs for Bitcoin and forkcoin nodes +# Copyright (C)2013-2025 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +test.cmdtest_d.httpd.ticker: Ticker WSGI http server +""" + +from . import HTTPD + +class TickerServer(HTTPD): + name = 'ticker server' + port = 19900 + content_type = 'application/json' + + def make_response_body(self, method, environ): + + with open(f'test/ref/ticker/ticker.json') as fh: + text = fh.read() + + return text.encode() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index fb48282..e9d4c32 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -16,6 +16,7 @@ import os, shutil from ..include.common import cfg from .base import CmdTestBase +from .httpd.ticker import TickerServer refdir = os.path.join('test','ref','ticker') @@ -70,8 +71,9 @@ class CmdTestScripts(CmdTestBase): ), 'ticker': ( "'mmnode-ticker' script", - ('ticker1', 'ticker [--help)'), + ('ticker1', 'ticker [--help]'), ('ticker2', 'ticker (bad proxy)'), + ('ticker2a', 'ticker [--download=cc]'), ('ticker3', 'ticker [--cached-data]'), ('ticker4', 'ticker [--cached-data --wide]'), ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), @@ -90,9 +92,12 @@ class CmdTestScripts(CmdTestBase): ) } - @property - def ticker_args(self): - return [ f'--cachedir={self.tmpdir}', '--proxy=http://asdfzxcv:32459' ] + def __init__(self, cfg, trunner, cfgs, spawn): + if not trunner: + return + self.ticker_server = TickerServer(cfg) + self.ticker_server.start() + return super().__init__(cfg, trunner, cfgs, spawn) @property def nt_datadir(self): @@ -100,7 +105,6 @@ class CmdTestScripts(CmdTestBase): def ticker_setup(self): self.spawn('',msg_only=True) - shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) @@ -111,11 +115,15 @@ class CmdTestScripts(CmdTestBase): args = [], expect_list = None, cached_data = True, + add_opts = [], + use_proxy = True, exit_val = None): t = self.spawn( 'mmnode-ticker', (['--cached-data'] if cached_data else []) - + self.ticker_args + + [f'--cachedir={self.tmpdir}'] + + (['--proxy=http://asdfzxcv:32459'] if use_proxy else []) + + add_opts + args, exit_val = exit_val) if expect_list: @@ -136,6 +144,13 @@ class CmdTestScripts(CmdTestBase): t.exit_val = 1 if ret else 3 return t + def ticker2a(self): + t = self.ticker( + add_opts = ['--proxy', '', '--download=cc'], + cached_data = False, + use_proxy = False) + return t + def ticker3(self): return self.ticker( [], diff --git a/test/init.sh b/test/init.sh index 134f4f3..eec6ec3 100755 --- a/test/init.sh +++ b/test/init.sh @@ -98,6 +98,7 @@ create_test_links() { test/modtest.py hard test/test-release.sh symbolic test/cmdtest_d/base.py symbolic + test/cmdtest_d/httpd/__init__.py symbolic test/cmdtest_d/include/common.py symbolic test/cmdtest_d/include/runner.py symbolic test/cmdtest_d/include/group_mgr.py symbolic diff --git a/test/overlay/fakemods/mmgen_node_tools/Ticker.py b/test/overlay/fakemods/mmgen_node_tools/Ticker.py new file mode 100644 index 0000000..8d1ede4 --- /dev/null +++ b/test/overlay/fakemods/mmgen_node_tools/Ticker.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# MMGen Node Tools, terminal-based programs for Bitcoin and forkcoin nodes +# Copyright (C)2013-2025 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +fakemods.mmgen_node_tools.Ticker: fake module for Ticker class +""" + +from .Ticker_orig import * + +class overlay_fake_DataSource: + class coinpaprika: + api_host = 'localhost:19900' + api_proto = 'http' + +DataSource.coinpaprika.api_host = overlay_fake_DataSource.coinpaprika.api_host +DataSource.coinpaprika.api_proto = overlay_fake_DataSource.coinpaprika.api_proto From a8adef0be54c752c10a8f34d594ed3faaa23de0c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 60/81] mmgen-ticker: various fixes and cleanups --- mmgen_node_tools/Ticker.py | 39 +++++++++++++++++++--------------- test/cmdtest_d/httpd/ticker.py | 2 +- test/cmdtest_d/misc.py | 36 +++++++++++++++---------------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bfe899b..9aee86c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen +from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -99,18 +99,21 @@ class DataSource: if not os.path.exists(cfg.cachedir): os.makedirs(cfg.cachedir) - if not os.path.exists(self.json_fn): - open(self.json_fn, 'w').write('{}') - use_cached_data = cfg.cached_data and not gcfg.download if use_cached_data: data_type = 'json' - data_in = open(self.json_fn).read() + try: + data_in = open(self.json_fn).read() + except FileNotFoundError: + die(1, f'Cannot use cached data, because {self.json_fn_disp} does not exist') else: data_type = self.net_data_type - elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) - if elapsed >= self.timeout or gcfg.testing: + try: + mtime = os.stat(self.json_fn).st_mtime + except FileNotFoundError: + mtime = 0 + if (elapsed := int(time.time() - mtime)) >= self.timeout or gcfg.testing: if gcfg.testing: msg('') self.fetch_delay() @@ -148,14 +151,14 @@ class DataSource: if use_cached_data: if not cfg.quiet: - msg(f'Using cached data from ~/{self.json_fn_rel}') + msg(f'Using cached data from {self.json_fn_disp}') else: if os.path.exists(self.json_fn): os.rename(self.json_fn, self.json_fn + '.bak') with open(self.json_fn, 'w') as fh: fh.write(json_text) if not cfg.quiet: - msg(f'JSON data cached to ~/{self.json_fn_rel}') + msg(f'JSON data cached to {self.json_fn_disp}') if gcfg.download: sys.exit(0) @@ -168,8 +171,8 @@ class DataSource: return data @property - def json_fn_rel(self): - return os.path.relpath(self.json_fn, start=homedir) + def json_fn_disp(self): + return '~/' + os.path.relpath(self.json_fn, start=homedir) class coinpaprika(base): desc = 'CoinPaprika' @@ -186,8 +189,9 @@ class DataSource: self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self, elapsed): + rem = self.timeout - elapsed return ( - f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + + f'Rate limit exceeded! Retry in {rem} second{suf(rem)}' + ('' if cfg.btc_only else ', or use --cached-data or --btc')) @property @@ -270,7 +274,8 @@ class DataSource: 'last_updated': data['regularMarketTime']} def rate_limit_errmsg(self, elapsed): - return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' + rem = self.timeout - elapsed + return f'Rate limit exceeded! Retry in {rem} second{suf(rem)}, or use --cached-data' @property def json_fn(self): @@ -364,10 +369,10 @@ def gen_data(data): checking for duplicates. """ - def dup_sym_errmsg(dup_sym): + def dup_sym_errmsg(data_type, dup_sym): return ( f'The symbol {dup_sym!r} is shared by the following assets:\n' + - '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + + '\n ' + '\n '.join(d['id'] for d in data[data_type].data if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + f'instead of {dup_sym!r}') @@ -421,7 +426,7 @@ def gen_data(data): if not isinstance(v, dict): die(2, str(v)) if id in found['id']: - die(1, dup_sym_errmsg(id)) + die(1, dup_sym_errmsg('fi', id)) if m := data['hi'].get(k): spot = v['regularMarketPrice']['raw'] hist = tuple(m.values()) @@ -444,7 +449,7 @@ def gen_data(data): if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: - die(1, dup_sym_errmsg(d[k])) + die(1, dup_sym_errmsg('cc', d[k])) if not 'price_usd' in d: d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd diff --git a/test/cmdtest_d/httpd/ticker.py b/test/cmdtest_d/httpd/ticker.py index 2d946ae..6131875 100755 --- a/test/cmdtest_d/httpd/ticker.py +++ b/test/cmdtest_d/httpd/ticker.py @@ -21,7 +21,7 @@ class TickerServer(HTTPD): def make_response_body(self, method, environ): - with open(f'test/ref/ticker/ticker.json') as fh: + with open('test/ref/ticker/ticker.json') as fh: text = fh.read() return text.encode() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index e9d4c32..1cde41e 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -61,19 +61,16 @@ class CmdTestScripts(CmdTestBase): color = True cmd_group_in = ( - ('subgroup.ticker_setup', []), - ('subgroup.ticker', ['ticker_setup']), + ('subgroup.ticker', []), ) cmd_subgroups = { - 'ticker_setup': ( - "setup for 'ticker' subgroup", - ('ticker_setup', 'ticker setup'), - ), 'ticker': ( "'mmnode-ticker' script", ('ticker1', 'ticker [--help]'), + ('copy_files', 'copying JSON files to cache'), + ('ticker1a', 'ticker [--download=cc] (early caching)'), + ('ticker1b', 'ticker [--download=cc] (late caching)'), ('ticker2', 'ticker (bad proxy)'), - ('ticker2a', 'ticker [--download=cc]'), ('ticker3', 'ticker [--cached-data]'), ('ticker4', 'ticker [--cached-data --wide]'), ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), @@ -103,7 +100,7 @@ class CmdTestScripts(CmdTestBase): def nt_datadir(self): return os.path.join( cfg.data_dir_root, 'node_tools' ) - def ticker_setup(self): + def copy_files(self): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) @@ -135,20 +132,23 @@ class CmdTestScripts(CmdTestBase): t.expect('USAGE:') return t - def ticker2(self): - t = self.ticker(cached_data=False) - if not cfg.skipping_deps: - t.expect('Creating') - t.expect('Creating') - ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) - t.exit_val = 1 if ret else 3 - return t - - def ticker2a(self): + def ticker1a(self, first_run=True): t = self.ticker( add_opts = ['--proxy', '', '--download=cc'], cached_data = False, use_proxy = False) + if first_run and not cfg.skipping_deps: + t.expect('Creating') + t.expect('Creating') + return t + + def ticker1b(self): + return self.ticker1a(first_run=False) + + def ticker2(self): + t = self.ticker(cached_data=False) + ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) + t.exit_val = 1 if ret else 3 return t def ticker3(self): From 3b8aede9bad83e4187e888d7be939ae909508075 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 61/81] Ticker.gen_data(): new `process_data` class --- mmgen_node_tools/Ticker.py | 135 ++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9aee86c..8580553 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf +from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -389,6 +389,73 @@ def gen_data(data): if error: die(1, 'Missing data, exiting') + class process_data: + + def cc(): + nonlocal btcusd + for d in data['cc']: + if d['id'] == 'btc-bitcoin': + btcusd = Decimal(str(d['quotes']['USD']['price'])) + break + else: + raise ValueError('malformed cryptocurrency data') + for k in ('id', 'symbol'): + for d in data['cc']: + if wants[k]: + if d[k] in wants[k]: + if d[k] in found[k]: + die(1, dup_sym_errmsg('cc', d[k])) + if not 'price_usd' in d: + d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) + d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd + d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] + d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] + d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] + d['last_updated'] = int(datetime.datetime.fromisoformat( + d['last_updated']).timestamp()) + yield (d['id'], d) + found[k].add(d[k]) + wants[k].remove(d[k]) + if d[k] in usr_rate_assets_want[k]: + rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key + else: + break + + def fi(): + get_id = src_cls['fi'].get_id + conv_func = src_cls['fi'].conv_data + for k, v in data['fi'].items(): + id = get_id(k, v) + if wants['id']: + if id in wants['id']: + if not isinstance(v, dict): + die(2, str(v)) + if id in found['id']: + die(1, dup_sym_errmsg('fi', id)) + if hist := hist_close.get(k): + spot = v['regularMarketPrice']['raw'] + v['pct_chg_1wk'] = (spot / hist.close_1wk - 1) * 100 + v['pct_chg_4wks'] = (spot / hist.close_4wks - 1) * 100 # 4 weeks ≈ 1 month + v['pct_chg_1y'] = (spot / hist.close_1y - 1) * 100 + else: + v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None + yield (id, conv_func(id, v, btcusd)) + found['id'].add(id) + wants['id'].remove(id) + if id in usr_rate_assets_want['id']: # NB: using symbol instead of ID for key: + rate_assets[k] = conv_func(id, v, btcusd) + else: + break + + def hi(): + ret = namedtuple('historical_closing_prices', ['close_1wk', 'close_4wks', 'close_1y']) + nonlocal hist_close + for k, v in data['hi'].items(): + hist = tuple(v.values()) + hist_close[k] = ret(hist[-2]['close'], hist[-5]['close'], hist[0]['close']) + return () + rows_want = { 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} @@ -411,62 +478,20 @@ def gen_data(data): wants = {k: rows_want[k] | usr_wants[k] for k in ('id', 'symbol')} - for d in data['cc']: - if d['id'] == 'btc-bitcoin': - btcusd = Decimal(str(d['quotes']['USD']['price'])) - break + btcusd = Decimal('1') # dummy + hist_close = {} - get_id = src_cls['fi'].get_id - conv_func = src_cls['fi'].conv_data + parse_fail = False + for data_type in ('cc', 'hi', 'fi'): # 'fi' depends on 'cc' and 'hi' so must go last + if data_type in data: + try: + yield from getattr(process_data, data_type)() + except Exception as e: + rmsg(f'Error in source data {data_type!r}: {e}') + parse_fail = True - for k, v in data['fi'].items(): - id = get_id(k, v) - if wants['id']: - if id in wants['id']: - if not isinstance(v, dict): - die(2, str(v)) - if id in found['id']: - die(1, dup_sym_errmsg('fi', id)) - if m := data['hi'].get(k): - spot = v['regularMarketPrice']['raw'] - hist = tuple(m.values()) - v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( - (spot / hist[-2]['close'] - 1) * 100, - (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month - (spot / hist[0]['close'] - 1) * 100) - else: - v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None - yield (id, conv_func(id, v, btcusd)) - found['id'].add(id) - wants['id'].remove(id) - if id in usr_rate_assets_want['id']: - rate_assets[k] = conv_func(id, v, btcusd) # NB: using symbol instead of ID for key - else: - break - - for k in ('id', 'symbol'): - for d in data['cc']: - if wants[k]: - if d[k] in wants[k]: - if d[k] in found[k]: - die(1, dup_sym_errmsg('cc', d[k])) - if not 'price_usd' in d: - d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) - d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd - d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] - d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] - d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] - d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] - # .replace('Z','+00:00') -- Python 3.9 backport - d['last_updated'] = int(datetime.datetime.fromisoformat( - d['last_updated'].replace('Z', '+00:00')).timestamp()) - yield (d['id'], d) - found[k].add(d[k]) - wants[k].remove(d[k]) - if d[k] in usr_rate_assets_want[k]: - rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key - else: - break + if parse_fail: + die(2, 'Invalid data encountered, exiting') check_assets_found(usr_wants, found) From e7de6890796505f2fa9499fae80c459455dc7aa2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 62/81] mmnode-ticker: cache network data after parsing --- mmgen_node_tools/Ticker.py | 48 ++++++++++++++++++++++++----------- mmgen_node_tools/data/version | 2 +- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 8580553..bbca8ef 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -19,7 +19,7 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys, os, re, time, datetime, json, yaml, random +import os, re, time, datetime, json, yaml, random from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple @@ -149,20 +149,18 @@ class DataSource: elif 'error' in data: die(1, data['error']) + self.data = self.postprocess_data(data) + if use_cached_data: + self.json_text = None if not cfg.quiet: msg(f'Using cached data from {self.json_fn_disp}') else: - if os.path.exists(self.json_fn): - os.rename(self.json_fn, self.json_fn + '.bak') - with open(self.json_fn, 'w') as fh: - fh.write(json_text) - if not cfg.quiet: - msg(f'JSON data cached to {self.json_fn_disp}') - if gcfg.download: - sys.exit(0) + self.json_text = json_text + if cache_data(self, no_overwrite=True): + self.json_text = None - return self.postprocess_data(data) + return self def json_data_error_msg(self, json_text): pass @@ -393,14 +391,14 @@ def gen_data(data): def cc(): nonlocal btcusd - for d in data['cc']: + for d in data['cc'].data: if d['id'] == 'btc-bitcoin': btcusd = Decimal(str(d['quotes']['USD']['price'])) break else: raise ValueError('malformed cryptocurrency data') for k in ('id', 'symbol'): - for d in data['cc']: + for d in data['cc'].data: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: @@ -425,7 +423,7 @@ def gen_data(data): def fi(): get_id = src_cls['fi'].get_id conv_func = src_cls['fi'].conv_data - for k, v in data['fi'].items(): + for k, v in data['fi'].data.items(): id = get_id(k, v) if wants['id']: if id in wants['id']: @@ -451,7 +449,7 @@ def gen_data(data): def hi(): ret = namedtuple('historical_closing_prices', ['close_1wk', 'close_4wks', 'close_1y']) nonlocal hist_close - for k, v in data['hi'].items(): + for k, v in data['hi'].data.items(): hist = tuple(v.values()) hist_close[k] = ret(hist[-2]['close'], hist[-5]['close'], hist[0]['close']) return () @@ -489,10 +487,15 @@ def gen_data(data): except Exception as e: rmsg(f'Error in source data {data_type!r}: {e}') parse_fail = True + else: + cache_data(data[data_type]) if parse_fail: die(2, 'Invalid data encountered, exiting') + if gcfg.download: + return + check_assets_found(usr_wants, found) for asset in (cfg.usr_rows + cfg.usr_columns): @@ -518,6 +521,18 @@ def gen_data(data): 'price_btc': Decimal(1) / btcusd, 'last_updated': None}) +def cache_data(data_src, no_overwrite=False): + if data_src.json_text: + if os.path.exists(data_src.json_fn): + if no_overwrite: + return False + os.rename(data_src.json_fn, data_src.json_fn + '.bak') + with open(data_src.json_fn, 'w') as fh: + fh.write(data_src.json_text) + if not cfg.quiet: + msg(f'JSON data cached to {data_src.json_fn_disp}') + return True + def main(): def update_sample_file(usr_cfg_file): @@ -558,7 +573,7 @@ def main(): return if gcfg.list_ids: - do_pager('\n'.join(e['id'] for e in src_data['cc'])) + do_pager('\n'.join(e.data['id'] for e in src_data['cc'])) return global now @@ -566,6 +581,9 @@ def main(): data = dict(gen_data(src_data)) + if gcfg.download: + return + (do_pager if cfg.pager else Msg_r)( '\n'.join(getattr(Ticker, cfg.clsname)(data).gen_output()) + '\n') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f9d163c..4e5cebc 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev3 +3.6.dev4 From 9ffaed6a912a4a42c84f5e9675a0d70d5edd9fae Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:58:54 +0000 Subject: [PATCH 63/81] mmnode-ticker: improve options setting Options set in the cfg file can now be unset on the command line, e.g.: $ mmnode-ticker --no-quiet --no-cached-data --- mmgen_node_tools/Ticker.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bbca8ef..49ce857 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -712,6 +712,12 @@ def make_cfg(gcfg_arg): rows += (hdr,) + uniq_data return rows + def get_cfg_var(name): + if name in gcfg._uopts: + return getattr(gcfg, name) + else: + return getattr(gcfg, name) or cfg_in.cfg.get(name) + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -772,27 +778,24 @@ def make_cfg(gcfg_arg): query = query, adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', - btc_only = gcfg.btc or cfg_in.cfg.get('btc'), - add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), - cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, + btc_only = get_cfg_var('btc'), + add_prec = parse_add_precision(get_cfg_var('add_precision')), + cachedir = get_cfg_var('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = - get_portfolio() - if cfg_in.portfolio - and (gcfg.portfolio or cfg_in.cfg.get('portfolio')) - and not query + get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query else None, - percent_cols = parse_percent_cols(gcfg.percent_cols or cfg_in.cfg.get('percent_cols')), - asset_limit = gcfg.asset_limit or cfg_in.cfg.get('asset_limit'), - cached_data = gcfg.cached_data or cfg_in.cfg.get('cached_data'), - elapsed = gcfg.elapsed or cfg_in.cfg.get('elapsed'), - name_labels = gcfg.name_labels or cfg_in.cfg.get('name_labels'), - pager = gcfg.pager or cfg_in.cfg.get('pager'), - thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), - update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), - quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), - verbose = gcfg.verbose or cfg_in.cfg.get('verbose')) + percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), + asset_limit = get_cfg_var('asset_limit'), + cached_data = get_cfg_var('cached_data'), + elapsed = get_cfg_var('elapsed'), + name_labels = get_cfg_var('name_labels'), + pager = get_cfg_var('pager'), + thousands_comma = get_cfg_var('thousands_comma'), + update_time = get_cfg_var('update_time'), + quiet = get_cfg_var('quiet'), + verbose = get_cfg_var('verbose')) def get_cfg_in(): ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) From 083b29eae829d24be0714355f759d1258300da87 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:59:00 +0000 Subject: [PATCH 64/81] mmnode-ticker: minor fixes and cleanups --- mmgen_node_tools/Ticker.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 49ce857..9d1c9df 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf +from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -184,7 +184,7 @@ class DataSource: dfl_asset_limit = 2000 def __init__(self): - self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) + self.asset_limit = int(cfg.asset_limit) if is_int(cfg.asset_limit) else self.dfl_asset_limit def rate_limit_errmsg(self, elapsed): rem = self.timeout - elapsed @@ -573,7 +573,7 @@ def main(): return if gcfg.list_ids: - do_pager('\n'.join(e.data['id'] for e in src_data['cc'])) + do_pager('\n'.join(e['id'] for e in src_data['cc'].data)) return global now @@ -606,7 +606,7 @@ def make_cfg(gcfg_arg): return tuple(gen()) def parse_percent_cols(arg): - if arg is None: + if arg is None or arg.lower() in ('none', ''): return [] res = arg.lower().split(',') for s in res: @@ -638,6 +638,8 @@ def make_cfg(gcfg_arg): source = parsed_id.source) cl_opt = getattr(gcfg, key) + if (cl_opt or '').lower() in ('', 'none'): + return () cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) @@ -718,6 +720,12 @@ def make_cfg(gcfg_arg): else: return getattr(gcfg, name) or cfg_in.cfg.get(name) + def get_proxy(name): + proxy = getattr(gcfg, name) + return ( + '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') + else (proxy or cfg_in.cfg.get(name))) + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -749,7 +757,6 @@ def make_cfg(gcfg_arg): src_cls = {k: getattr(DataSource, v) for k, v in DataSource.get_sources().items()} fi_pat = src_cls['fi'].asset_id_pat - cmd_args = gcfg._args cfg_in = get_cfg_in() if gcfg.test_suite: # required for testing with overlay @@ -757,15 +764,15 @@ def make_cfg(gcfg_arg): this_mod.src_cls = src_cls this_mod.cfg_in = cfg_in + if cmd_args := gcfg._args: + if len(cmd_args) > 1: + die(1, 'Only one command-line argument is allowed') + query = parse_query_arg(cmd_args[0]) + else: + query = None + usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) - query = parse_query_arg(cmd_args[0]) if cmd_args else None - - def get_proxy(name): - proxy = getattr(gcfg, name) - return ( - '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') - else (proxy or cfg_in.cfg.get(name))) proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy From 060b968ad4b88f845b9cd95fa89174a69b4b9743 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:59:00 +0000 Subject: [PATCH 65/81] mmnode-ticker: display crypto assets by market cap Examples: # Display top 2000 assets by market cap: $ mmnode-ticker 2000 # Display assets 201-300 by market cap, displaying all available columns: $ mmnode-ticker --widest 201-300 # Display asset 32 by market cap: $ mmnode-ticker 32-32 Testing/demo: $ test/cmdtest.py -e scripts.ticker --- mmgen_node_tools/Ticker.py | 53 +++++++++++++++++++++++++++++---- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 25 +++++++++++----- test/cmdtest_d/misc.py | 42 ++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9d1c9df..bf3398b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -182,6 +182,7 @@ class DataSource: net_data_type = 'json' has_verbose = True dfl_asset_limit = 2000 + max_asset_idx = 1_000_000 def __init__(self): self.asset_limit = int(cfg.asset_limit) if is_int(cfg.asset_limit) else self.dfl_asset_limit @@ -576,6 +577,17 @@ def main(): do_pager('\n'.join(e['id'] for e in src_data['cc'].data)) return + global cfg + + if cfg.asset_range: + func = DataSource.coinpaprika.parse_asset_id + n, m = cfg.asset_range + cfg = cfg._replace(rows = + tuple(func(e['id'], require_label=False) for e in src_data['cc'].data[n-1:m]) + + tuple(['-']) + + tuple([func('btc-bitcoin', require_label=True)]) + + tuple(r for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi')) + global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 @@ -643,6 +655,21 @@ def make_cfg(gcfg_arg): cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) + def parse_asset_range(s): + max_idx = DataSource.coinpaprika.max_asset_idx + match s.split('-'): + case [a, b] if is_int(a) and is_int(b): + n, m = (int(a), int(b)) + case [a] if is_int(a): + n, m = (1, int(a)) + case _: + return None + if n < 1 or m < 1 or n > m: + raise ValueError(f'‘{s}’: invalid asset range specifier') + if m > max_idx: + raise ValueError(f'‘{s}’: end of range must be <= {max_idx}') + return (n, m) + def parse_query_arg(s): """ asset_id:amount[:to_asset_id[:to_amount]] @@ -731,6 +758,7 @@ def make_cfg(gcfg_arg): 'usr_rows', 'usr_columns', 'query', + 'asset_range', 'adjust', 'clsname', 'btc_only', @@ -767,8 +795,10 @@ def make_cfg(gcfg_arg): if cmd_args := gcfg._args: if len(cmd_args) > 1: die(1, 'Only one command-line argument is allowed') - query = parse_query_arg(cmd_args[0]) + asset_range = parse_asset_range(cmd_args[0]) + query = None if asset_range else parse_query_arg(cmd_args[0]) else: + asset_range = None query = None usr_rows = parse_usr_asset_arg('add_rows') @@ -783,6 +813,7 @@ def make_cfg(gcfg_arg): usr_rows = usr_rows, usr_columns = usr_columns, query = query, + asset_range = asset_range, adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', btc_only = get_cfg_var('btc'), @@ -930,12 +961,14 @@ class Ticker: if self.table_hdr: yield self.table_hdr - for row in self.rows: + for n, row in enumerate(self.rows, cfg.asset_range[0] if cfg.asset_range else 1): if isinstance(row, str): + if cfg.asset_range: + return yield ('-' * self.hl_wid) else: try: - yield self.fmt_row(self.data[row.id]) + yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') @@ -989,7 +1022,7 @@ class Ticker: d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids} - def fmt_row(self, d, amt=None, amt_fmt=None): + def fmt_row(self, d, amt=None, amt_fmt=None, idx=None): def fmt_pct(n): return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') @@ -1002,6 +1035,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( + idx = idx, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), @@ -1061,6 +1095,15 @@ class Ticker: self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) + if cfg.asset_range: + def get_col1_w(): + for n, r in enumerate(cfg.rows): + if isinstance(r, str): + return len(str(n)) + col1_w = get_col1_w() + self.fs_str = ' ' * (col1_w + 2) + self.fs_str + self.fs_num = f'{{idx:{col1_w}}}) ' + self.fs_num + @property def table_hdr(self): return self.fs_str.format( @@ -1136,7 +1179,7 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self, d): + def fmt_row(self, d, idx=None): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4e5cebc..636c831 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev4 +3.6.dev5 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c36d886..8efb97a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -25,7 +25,7 @@ opts_data = { ], 'text': { 'desc': 'Display prices for cryptocurrency and other assets', - 'usage': '[opts] [TRADE_SPECIFIER]', + 'usage': '[opts] [TRADE_SPECIFIER | ASSET_RANGE]', 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) @@ -71,14 +71,19 @@ opts_data = { """, 'notes': """ -The script has two display modes: ‘overview’, the default, and ‘trading’, the -latter being enabled when a TRADE_SPECIFIER argument (see below) is supplied -on the command line. +The script has three display modes: ‘overview’, enabled when no arguments are +given on the command line; ‘trading’, when a TRADE_SPECIFIER argument (see +below) is given; and ‘market cap’, when an ASSET_RANGE (see below) is given. Overview mode displays prices of all configured assets, and optionally the -user’s portfolio, while trading mode displays the price of a given quantity -of an asset in relation to other assets, optionally comparing an offered -price to the spot price. +user’s portfolio; trading mode displays the price of a given quantity of an +asset in relation to other assets, optionally comparing an offered price to +the spot price; and market cap mode lists a range of crypto assets selected +by current market cap. + +The ASSET_RANGE argument can be either an integer N, in which case the top +N assets by market cap will be displayed, or a hyphen-separated range N-M, +in which case assets from N to M by market cap will be displayed. ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids) consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the @@ -199,6 +204,12 @@ $ mmnode-ticker usd:2700:btc:0.123 # current spot price, at specified USDINR rate: $ mmnode-ticker -n -c inr-indian-rupee:79.5 inr:200000:btc:0.1 +# Display top 20 crypto assets by market cap, adding a Euro column: +$ mmnode-ticker -c eurusd=x 20 + +# Same as above, specifying assets using a range: +$ mmnode-ticker -c eurusd=x 1-20 + CONFIGURED ASSETS: {assets} diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 1cde41e..c2dceff 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -86,6 +86,10 @@ class CmdTestScripts(CmdTestBase): ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), ('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'), + ('ticker18', 'ticker [--cached-data --widest --add-columns eurusd=x 10]'), + ('ticker19', 'ticker [--cached-data 1-5]'), + ('ticker20', 'ticker [--cached-data 2-5]'), + ('ticker21', 'ticker [--cached-data 5-5]'), ) } @@ -319,3 +323,41 @@ class CmdTestScripts(CmdTestBase): 'BITCOIN 23,250.77 42,731.767 1.00000000', 'BULGARIAN LEV 0.54 1.000 0.00002340', ]) + + def ticker18(self): + return self.ticker( + ['10'], + [ + r'1\) BITCOIN 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', + r'6\) ALGORAND 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + ], + add_opts = ['--widest', '--add-columns=eurusd=x']) + + def ticker19(self): + return self.ticker( + ['1-5'], + [ + 'USD EURUSD=X BTC ' + r'1\) BTC 23250.77 21848.7527 1.00000000', + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) + + def ticker20(self): + return self.ticker( + ['2-5'], + [ + 'USD EURUSD=X BTC ' + r'2\) ETH 1659.66 1559.5846 0.07138094', + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) + + def ticker21(self): + return self.ticker( + ['5-5'], + [ + 'USD EURUSD=X BTC ' + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) From c1f42fc25b123c334262e8cb53b460753e1c0be2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:14 +0000 Subject: [PATCH 66/81] mmnode-ticker: various fixes and cleanups --- mmgen_node_tools/Ticker.py | 135 +++++++++++++++++--------------- mmgen_node_tools/main_ticker.py | 4 +- test/cmdtest_d/misc.py | 6 +- 3 files changed, 81 insertions(+), 64 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bf3398b..7122e44 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -607,12 +607,10 @@ def make_cfg(gcfg_arg): def parse_asset_id(s, require_label=False): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) - def get_rows_from_cfg(add_data=None): + def get_rows_from_cfg(): def gen(): - for n, (k, v) in enumerate(cfg_in.cfg['assets'].items()): + for k, v in cfg_in.cfg['assets'].items(): yield k - if add_data and k in add_data: - v += tuple(add_data[k]) for e in v: yield parse_asset_id(e, require_label=True) return tuple(gen()) @@ -650,7 +648,7 @@ def make_cfg(gcfg_arg): source = parsed_id.source) cl_opt = getattr(gcfg, key) - if (cl_opt or '').lower() in ('', 'none'): + if cl_opt is None or cl_opt.lower() in ('none', ''): return () cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) @@ -702,15 +700,16 @@ def make_cfg(gcfg_arg): def get_usr_assets(): return ( - 'user_added', - usr_rows + - (tuple(asset for asset in query if asset) if query else ()) + - usr_columns) + usr_rows + + (tuple(asset for asset in query if asset) if query else ()) + + usr_columns) - def get_portfolio_assets(ret=()): + def get_portfolio_assets(): if cfg_in.portfolio and gcfg.portfolio: ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) - return ('portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC')) + return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') + else: + return () def get_portfolio(): return {k: Decimal(v) for k, v in cfg_in.portfolio.items() @@ -730,14 +729,12 @@ def make_cfg(gcfg_arg): rows = ( ('trade_pair',) + query if (query and query.to_asset) else ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg(add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None)) - + get_rows_from_cfg()) for hdr, data in ( - (get_usr_assets(),) if query else - (get_usr_assets(), get_portfolio_assets())): + ('user_uniq', get_usr_assets()), + ('portfolio_uniq', get_portfolio_assets())): if data: - uniq_data = tuple(gen_uniq(data, 'symbol', preload=rows)) - if uniq_data: + if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): rows += (hdr,) + uniq_data return rows @@ -787,11 +784,6 @@ def make_cfg(gcfg_arg): cfg_in = get_cfg_in() - if gcfg.test_suite: # required for testing with overlay - from . import Ticker as this_mod - this_mod.src_cls = src_cls - this_mod.cfg_in = cfg_in - if cmd_args := gcfg._args: if len(cmd_args) > 1: die(1, 'Only one command-line argument is allowed') @@ -808,6 +800,13 @@ def make_cfg(gcfg_arg): proxy = None if proxy == 'none' else proxy proxy2 = get_proxy('proxy2') + portfolio = ( + get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query + else None) + + if portfolio and asset_range: + die(1, '--portfolio not supported in market cap view') + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, @@ -821,9 +820,7 @@ def make_cfg(gcfg_arg): cachedir = get_cfg_var('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = - get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query - else None, + portfolio = portfolio, percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), @@ -835,6 +832,8 @@ def make_cfg(gcfg_arg): quiet = get_cfg_var('quiet'), verbose = get_cfg_var('verbose')) + return (src_cls, cfg_in) + def get_cfg_in(): ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) cfg_file, portfolio_file = ( @@ -871,7 +870,7 @@ class Ticker: self.col1_wid = max(len('TOTAL'), ( max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else - max(len(d['symbol']) for d in data.values()))) + 1 + max(len(d['symbol']) for d in data.values()))) self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row for row in cfg.rows] @@ -881,7 +880,7 @@ class Ticker: for row in self.rows if isinstance(row, tuple) and row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') - def format_last_update_col(self, cross_assets=()): + def format_last_updated_col(self, cross_assets=()): if cfg.elapsed: from mmgen.util2 import format_elapsed_hr @@ -899,19 +898,20 @@ class Ticker: min_t = None for row in self.rows: - if isinstance(row, tuple): - try: - t = int(d[row.id]['last_updated']) - except TypeError as e: - d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) - except KeyError as e: - msg(str(e)) - pass - else: - t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( - (min(t, min_t) if min_t else t), - now = now) - max_w = max(len(t_fmt), max_w) + if not isinstance(row, tuple): + continue + try: + t = int(d[row.id]['last_updated']) + except TypeError as e: + d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) + except KeyError as e: + msg(str(e)) + pass + else: + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( + (min(t, min_t) if min_t else t), + now = now) + max_w = max(len(t_fmt), max_w) self.upd_w = max_w @@ -924,8 +924,9 @@ class Ticker: if asset.id: return asset.id else: + m = asset.symbol for d in self.data.values(): - if d['symbol'] == asset.symbol: + if m == d['symbol']: return d['id'] def create_label(self, id): @@ -961,16 +962,26 @@ class Ticker: if self.table_hdr: yield self.table_hdr - for n, row in enumerate(self.rows, cfg.asset_range[0] if cfg.asset_range else 1): - if isinstance(row, str): - if cfg.asset_range: - return - yield ('-' * self.hl_wid) - else: + if cfg.asset_range: + yield '-' * self.hl_wid + for n, row in enumerate(self.rows, cfg.asset_range[0]): + if isinstance(row, str): + break try: yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') + else: + for row in self.rows: + if isinstance(row, str): + if cfg.asset_range: + return + yield ('-' * self.hl_wid) + else: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') yield '-' * self.hl_wid @@ -1003,7 +1014,7 @@ class Ticker: super().__init__(data) - self.format_last_update_col() + self.format_last_updated_col() if cfg.portfolio: self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] @@ -1072,9 +1083,9 @@ class Ticker: ) for k in self.col_ids} cols = ( - ['label', 'usd-us-dollar'] + - [asset.id for asset in self.usr_col_assets] + - [a for a, b in ( + ['label', 'usd-us-dollar'] + + [asset.id for asset in self.usr_col_assets] + + [a for a, b in ( ('btc-bitcoin', not cfg.btc_only), ('pct1y', 'y' in cfg.percent_cols), ('pct1m', 'm' in cfg.percent_cols), @@ -1082,6 +1093,17 @@ class Ticker: ('pct1d', 'd' in cfg.percent_cols), ('update_time', cfg.update_time)) if b]) + + if cfg.asset_range: + def get_num_w(): + for n, r in enumerate(cfg.rows): + if isinstance(r, str): + return len(str(n)) + num_w = get_num_w() + col_fs_data.update({ + 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) + cols = ['idx'] + cols + cols2 = list(cols) if cfg.update_time: cols2.pop() @@ -1095,15 +1117,6 @@ class Ticker: self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) - if cfg.asset_range: - def get_col1_w(): - for n, r in enumerate(cfg.rows): - if isinstance(r, str): - return len(str(n)) - col1_w = get_col1_w() - self.fs_str = ' ' * (col1_w + 2) + self.fs_str - self.fs_num = f'{{idx:{col1_w}}}) ' + self.fs_num - @property def table_hdr(self): return self.fs_str.format( @@ -1152,7 +1165,7 @@ class Ticker: for a in self.usr_col_assets: self.prices[a.id]['usd-us-dollar'] = data[a.id]['price_usd'] - self.format_last_update_col(cross_assets=self.usr_col_assets) + self.format_last_updated_col(cross_assets=self.usr_col_assets) self.init_prec() self.init_fs() diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 8efb97a..c6b41b3 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -244,9 +244,9 @@ from . import Ticker gcfg = Config(opts_data=opts_data, caller_post_init=True) -Ticker.make_cfg(gcfg) +src_cls, cfg_in = Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, cfg_in, src_cls +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen gcfg._post_init() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index c2dceff..b64e33c 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -338,8 +338,10 @@ class CmdTestScripts(CmdTestBase): ['1-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'1\) BTC 23250.77 21848.7527 1.00000000', - r'5\) ADA 0.51 0.4764 0.00002180', + r'5\) ADA 0.51 0.4764 0.00002180' + ' ---------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -348,6 +350,7 @@ class CmdTestScripts(CmdTestBase): ['2-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'2\) ETH 1659.66 1559.5846 0.07138094', r'5\) ADA 0.51 0.4764 0.00002180', ], @@ -358,6 +361,7 @@ class CmdTestScripts(CmdTestBase): ['5-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'5\) ADA 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From 3f921d333ccba378a6309123214e0a851e707ae8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 67/81] Ticker.py: `parse_asset_id()`: unify call signature --- mmgen_node_tools/Ticker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7122e44..76d9d2a 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -233,7 +233,7 @@ class DataSource: return [data] if cfg.btc_only else data @staticmethod - def parse_asset_id(s, require_label): + def parse_asset_id(s, require_label=True): sym, label = (*s.split('-', 1), None)[:2] if require_label and not label: die(1, f'{s!r}: asset label is missing') @@ -314,7 +314,7 @@ class DataSource: return ticker.price @staticmethod - def parse_asset_id(s, require_label): + def parse_asset_id(s, require_label=True): return asset_tuple( symbol = s.upper(), id = s.lower(), @@ -604,7 +604,7 @@ def make_cfg(gcfg_arg): query_tuple = namedtuple('query', ['asset', 'to_asset']) asset_data = namedtuple('asset_data', ['symbol', 'id', 'amount', 'rate', 'rate_asset', 'source']) - def parse_asset_id(s, require_label=False): + def parse_asset_id(s, require_label=True): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) def get_rows_from_cfg(): @@ -634,7 +634,7 @@ def make_cfg(gcfg_arg): ss = s.split(':') assert len(ss) in (1, 2, 3), f'{s}: malformed argument' asset_id, rate, rate_asset = (*ss, None, None)[:3] - parsed_id = parse_asset_id(asset_id) + parsed_id = parse_asset_id(asset_id, require_label=False) return asset_data( symbol = parsed_id.symbol, @@ -644,7 +644,7 @@ def make_cfg(gcfg_arg): None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else Decimal(rate)), - rate_asset = parse_asset_id(rate_asset) if rate_asset else None, + rate_asset = parse_asset_id(rate_asset, require_label=False) if rate_asset else None, source = parsed_id.source) cl_opt = getattr(gcfg, key) @@ -673,7 +673,7 @@ def make_cfg(gcfg_arg): asset_id:amount[:to_asset_id[:to_amount]] """ def parse_query_asset(asset_id, amount): - parsed_id = parse_asset_id(asset_id) + parsed_id = parse_asset_id(asset_id, require_label=False) return asset_data( symbol = parsed_id.symbol, id = parsed_id.id, @@ -706,7 +706,7 @@ def make_cfg(gcfg_arg): def get_portfolio_assets(): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) + ret = (parse_asset_id(e) for e in cfg_in.portfolio) return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') else: return () From 253aa14a265f66de3b97f826c0bef4d5ff3a369d Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 68/81] Ticker.py: new `RowDict` class --- mmgen_node_tools/Ticker.py | 86 +++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 48 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 76d9d2a..ed68eda 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -42,6 +42,11 @@ percent_cols = { 'y': 'year', } +class RowDict(dict): + + def __iter__(self): + return (e for v in self.values() for e in v) + class DataSource: source_groups = [ @@ -286,7 +291,7 @@ class DataSource: @property def symbols(self): - return [r.symbol for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi'] + return [r.symbol for r in cfg.rows if r.source == 'fi'] def get_data_from_network(self): @@ -456,8 +461,8 @@ def gen_data(data): return () rows_want = { - 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, - 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} + 'id': {r.id for r in cfg.rows if r.id} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if r.id is None} - {'USD'}} usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) usr_rate_assets_want = { 'id': {a.id for a in usr_rate_assets if a.id}, @@ -580,13 +585,16 @@ def main(): global cfg if cfg.asset_range: - func = DataSource.coinpaprika.parse_asset_id n, m = cfg.asset_range - cfg = cfg._replace(rows = - tuple(func(e['id'], require_label=False) for e in src_data['cc'].data[n-1:m]) - + tuple(['-']) - + tuple([func('btc-bitcoin', require_label=True)]) - + tuple(r for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi')) + cfg = cfg._replace(rows = RowDict({ + 'asset_list': + tuple( + asset_tuple(e['symbol'], e['id'], source='cc') + for e in src_data['cc'].data[n-1:m]), + 'extra': + tuple( + [asset_tuple('BTC', 'btc-bitcoin', source='cc')] + + [r for r in cfg.rows if r.source == 'fi'])})) global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 @@ -607,14 +615,6 @@ def make_cfg(gcfg_arg): def parse_asset_id(s, require_label=True): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) - def get_rows_from_cfg(): - def gen(): - for k, v in cfg_in.cfg['assets'].items(): - yield k - for e in v: - yield parse_asset_id(e, require_label=True) - return tuple(gen()) - def parse_percent_cols(arg): if arg is None or arg.lower() in ('none', ''): return [] @@ -726,16 +726,18 @@ def make_cfg(gcfg_arg): return int(s) def create_rows(): - rows = ( - ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg()) + rows = RowDict( + {'trade_pair': query} if (query and query.to_asset) else + {'bitcoin': [parse_asset_id('btc-bitcoin')]} if gcfg.btc else + {k: tuple(parse_asset_id(e) for e in v) for k, v in cfg_in.cfg['assets'].items()}) for hdr, data in ( ('user_uniq', get_usr_assets()), ('portfolio_uniq', get_portfolio_assets())): if data: if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): - rows += (hdr,) + uniq_data + rows[hdr] = uniq_data + else: + rows[hdr] = () return rows def get_cfg_var(name): @@ -872,12 +874,10 @@ class Ticker: max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else max(len(d['symbol']) for d in data.values()))) - self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row - for row in cfg.rows] + self.rows = RowDict( + {k: tuple(row._replace(id=self.get_id(row)) for row in v) for k, v in cfg.rows.items()}) self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} - - self.prices = {row.id: self.get_row_prices(row.id) - for row in self.rows if isinstance(row, tuple) and row.id in data} + self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') def format_last_updated_col(self, cross_assets=()): @@ -898,8 +898,6 @@ class Ticker: min_t = None for row in self.rows: - if not isinstance(row, tuple): - continue try: t = int(d[row.id]['last_updated']) except TypeError as e: @@ -964,24 +962,20 @@ class Ticker: if cfg.asset_range: yield '-' * self.hl_wid - for n, row in enumerate(self.rows, cfg.asset_range[0]): - if isinstance(row, str): - break + for n, row in enumerate(self.rows['asset_list'], cfg.asset_range[0]): try: yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') else: - for row in self.rows: - if isinstance(row, str): - if cfg.asset_range: - return - yield ('-' * self.hl_wid) - else: - try: - yield self.fmt_row(self.data[row.id]) - except KeyError: - yield gray(f'(no data for {row.id})') + for rows in self.rows.values(): + if rows: + yield '-' * self.hl_wid + for row in rows: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') yield '-' * self.hl_wid @@ -1019,7 +1013,7 @@ class Ticker: if cfg.portfolio: self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] for row in self.rows - if isinstance(row, tuple) and row.id in cfg.portfolio and row.id in data) + if row.id in cfg.portfolio and row.id in data) for col_id in self.col_ids} self.init_prec() @@ -1095,11 +1089,7 @@ class Ticker: if b]) if cfg.asset_range: - def get_num_w(): - for n, r in enumerate(cfg.rows): - if isinstance(r, str): - return len(str(n)) - num_w = get_num_w() + num_w = len(str(len(cfg.rows['asset_list']))) col_fs_data.update({ 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) cols = ['idx'] + cols From 9aa4b4dcfef0d95e96513042e42cc86c6bc997ba Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 69/81] mmnode-ticker: add `MarketCap` column Column is enabled automatically when script is invoked in market cap mode. --- mmgen_node_tools/Ticker.py | 8 ++++++-- mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 28 ++++++++++++++-------------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index ed68eda..9e598de 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -416,6 +416,7 @@ def gen_data(data): d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] + d['market_cap'] = d['quotes']['USD']['market_cap'] d['last_updated'] = int(datetime.datetime.fromisoformat( d['last_updated']).timestamp()) yield (d['id'], d) @@ -1041,6 +1042,7 @@ class Ticker: return self.fs_num.format( idx = idx, + mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), @@ -1091,8 +1093,9 @@ class Ticker: if cfg.asset_range: num_w = len(str(len(cfg.rows['asset_list']))) col_fs_data.update({ - 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) - cols = ['idx'] + cols + 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2), + 'mcap': fd('{mcap:>12}', '{mcap:12.5f}', 12)}) + cols = ['idx', 'label', 'mcap'] + cols[1:] cols2 = list(cols) if cfg.update_time: @@ -1111,6 +1114,7 @@ class Ticker: def table_hdr(self): return self.fs_str.format( lbl = '', + mcap = 'MarketCap(B)', pc1 = ' CHG_7d', pc2 = 'CHG_24h', pc3 = 'CHG_1y', diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 636c831..921f776 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev5 +3.6.dev6 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index b64e33c..a8d259e 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -328,8 +328,8 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['10'], [ - r'1\) BITCOIN 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', - r'6\) ALGORAND 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + r'1\) BITCOIN 444.33652 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', + r'6\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' ], add_opts = ['--widest', '--add-columns=eurusd=x']) @@ -337,11 +337,11 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['1-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'1\) BTC 23250.77 21848.7527 1.00000000', - r'5\) ADA 0.51 0.4764 0.00002180' - ' ---------------------------------------------' + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'1\) BTC 444.33652 23250.77 21848.7527 1.00000000', + r'5\) ADA 17.11161 0.51 0.4764 0.00002180' + ' ---------------------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -349,10 +349,10 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['2-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'2\) ETH 1659.66 1559.5846 0.07138094', - r'5\) ADA 0.51 0.4764 0.00002180', + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'2\) ETH 202.15129 1659.66 1559.5846 0.07138094', + r'5\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) @@ -360,8 +360,8 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['5-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'5\) ADA 0.51 0.4764 0.00002180', + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'5\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From e71ef141bffd8f44998e1510edf197f423cb2e3b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:09 +0000 Subject: [PATCH 70/81] mmnode-ticker: minor cleanups --- mmgen_node_tools/Ticker.py | 20 ++++++++++---------- mmgen_node_tools/main_ticker.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9e598de..6aef36f 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red, yellow, green, blue, orange, gray +from mmgen.color import red, yellow, green, blue, orange, gray, cyan from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager @@ -39,8 +39,7 @@ percent_cols = { 'd': 'day', 'w': 'week', 'm': 'month', - 'y': 'year', -} + 'y': 'year'} class RowDict(dict): @@ -713,8 +712,8 @@ def make_cfg(gcfg_arg): return () def get_portfolio(): - return {k: Decimal(v) for k, v in cfg_in.portfolio.items() - if (not gcfg.btc) or k == 'btc-bitcoin'} + return tuple((k, Decimal(v)) for k, v in cfg_in.portfolio.items() + if (not gcfg.btc) or k == 'btc-bitcoin') def parse_add_precision(arg): if not arg: @@ -932,7 +931,7 @@ class Ticker: return self.data[id]['name'].upper() def gen_output(self): - yield 'Current time: {} UTC'.format(time.strftime('%F %X', time.gmtime(now))) + yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) for asset in self.usr_col_assets: if asset.symbol != 'USD': @@ -987,7 +986,7 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym, amt in cfg.portfolio.items(): + for sym, amt in cfg.portfolio: try: yield self.fmt_row(self.data[sym], amt=amt) except KeyError: @@ -1005,16 +1004,17 @@ class Ticker: self.adjust = cfg.adjust self.show_adj = self.adjust != 1 self.usr_col_assets = [asset._replace(id=self.get_id(asset)) for asset in cfg.usr_columns] - self.col_ids = ('usd-us-dollar',) + tuple(a.id for a in self.usr_col_assets) + ('btc-bitcoin',) + self.col_ids = ('usd-us-dollar', 'btc-bitcoin') + tuple(a.id for a in self.usr_col_assets) super().__init__(data) self.format_last_updated_col() if cfg.portfolio: - self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + pf_dict = dict(cfg.portfolio) + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * pf_dict[row.id] for row in self.rows - if row.id in cfg.portfolio and row.id in data) + if row.id in pf_dict and row.id in data) for col_id in self.col_ids} self.init_prec() diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c6b41b3..e533e62 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -225,7 +225,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, - pc = fmt_list(Ticker.percent_cols, fmt='bare')), + pc = fmt_list(Ticker.percent_cols, fmt='fancy')), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), cfg = os.path.relpath(cfg_in.cfg_file, start=homedir), From e6d62fd18b2d9e5885ce4452c843850871550f60 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:15 +0000 Subject: [PATCH 71/81] mmnode-ticker: display coin ranking in first column --- mmgen_node_tools/Ticker.py | 40 +++++++++++++++++++++++--------------- test/cmdtest_d/misc.py | 14 ++++++------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6aef36f..a8b6595 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -865,6 +865,7 @@ class Ticker: offer = None to_asset = None + hidden_groups = ('extra',) def __init__(self, data): @@ -876,6 +877,14 @@ class Ticker: self.rows = RowDict( {k: tuple(row._replace(id=self.get_id(row)) for row in v) for k, v in cfg.rows.items()}) + + if cfg.asset_range: + self.max_rank = 0 + for group, rows in self.rows.items(): + if group not in self.hidden_groups: + for row in rows: + self.max_rank = max(self.max_rank, int(data[row.id]['rank'])) + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -931,6 +940,15 @@ class Ticker: return self.data[id]['name'].upper() def gen_output(self): + + def process_rows(rows): + yield '-' * self.hl_wid + for row in rows: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') + yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) for asset in self.usr_col_assets: @@ -961,21 +979,11 @@ class Ticker: yield self.table_hdr if cfg.asset_range: - yield '-' * self.hl_wid - for n, row in enumerate(self.rows['asset_list'], cfg.asset_range[0]): - try: - yield self.fmt_row(self.data[row.id], idx=n) - except KeyError: - yield gray(f'(no data for {row.id})') + yield from process_rows(self.rows['asset_list']) else: for rows in self.rows.values(): if rows: - yield '-' * self.hl_wid - for row in rows: - try: - yield self.fmt_row(self.data[row.id]) - except KeyError: - yield gray(f'(no data for {row.id})') + yield from process_rows(rows) yield '-' * self.hl_wid @@ -1028,7 +1036,7 @@ class Ticker: d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids} - def fmt_row(self, d, amt=None, amt_fmt=None, idx=None): + def fmt_row(self, d, amt=None, amt_fmt=None): def fmt_pct(n): return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') @@ -1041,7 +1049,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( - idx = idx, + idx = int(d['rank']) if cfg.asset_range else None, mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), @@ -1091,7 +1099,7 @@ class Ticker: if b]) if cfg.asset_range: - num_w = len(str(len(cfg.rows['asset_list']))) + num_w = len(str(self.max_rank)) col_fs_data.update({ 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2), 'mcap': fd('{mcap:>12}', '{mcap:12.5f}', 12)}) @@ -1186,7 +1194,7 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self, d, idx=None): + def fmt_row(self, d): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index a8d259e..3e909f4 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -329,7 +329,7 @@ class CmdTestScripts(CmdTestBase): ['10'], [ r'1\) BITCOIN 444.33652 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', - r'6\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + r'33\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' ], add_opts = ['--widest', '--add-columns=eurusd=x']) @@ -338,10 +338,10 @@ class CmdTestScripts(CmdTestBase): ['1-5'], [ r'MarketCap\(B\) USD EURUSD=X BTC ' - '--------------------------------------------------------- ' + '---------------------------------------------------------- ' r'1\) BTC 444.33652 23250.77 21848.7527 1.00000000', - r'5\) ADA 17.11161 0.51 0.4764 0.00002180' - ' ---------------------------------------------------------' + r'8\) ADA 17.11161 0.51 0.4764 0.00002180' + ' ----------------------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -350,9 +350,9 @@ class CmdTestScripts(CmdTestBase): ['2-5'], [ r'MarketCap\(B\) USD EURUSD=X BTC ' - '--------------------------------------------------------- ' + '---------------------------------------------------------- ' r'2\) ETH 202.15129 1659.66 1559.5846 0.07138094', - r'5\) ADA 17.11161 0.51 0.4764 0.00002180', + r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) @@ -362,6 +362,6 @@ class CmdTestScripts(CmdTestBase): [ r'MarketCap\(B\) USD EURUSD=X BTC ' '--------------------------------------------------------- ' - r'5\) ADA 17.11161 0.51 0.4764 0.00002180', + r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From 0a953e3ca039040b9ca4dbff501913bc39b760bb Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:15 +0000 Subject: [PATCH 72/81] mmnode-ticker: sort output by various parameters Supported parameters: d - 1-day % change w - 1-week % change m - 1-month % change y - 1-year % change p - asset price c - market cap Examples: # Display top 50 assets by market cap, sorting by price change # in last 24 hours: $ mmnode-ticker --sort=d 50 --- mmgen_node_tools/Ticker.py | 45 ++++++++++++++++++++++- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 11 +++++- test/cmdtest_d/misc.py | 63 +++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index a8b6595..d3045d4 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red, yellow, green, blue, orange, gray, cyan +from mmgen.color import red, yellow, green, blue, orange, gray, cyan, pink from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager @@ -41,6 +41,15 @@ percent_cols = { 'm': 'month', 'y': 'year'} +sp = namedtuple('sort_parameter', ['key', 'desc']) +sort_params = { + 'd': sp('percent_change_24h', '1-day % change'), + 'w': sp('percent_change_7d', '1-week % change'), + 'm': sp('percent_change_30d', '1-month % change'), + 'y': sp('percent_change_1y', '1-year % change'), + 'p': sp('price_usd', 'asset price'), + 'c': sp('market_cap', 'market cap')} + class RowDict(dict): def __iter__(self): @@ -274,6 +283,7 @@ class DataSource: 'percent_change_30d': data['pct_chg_4wks'], 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, + 'market_cap': 0, # dummy - required for sorting 'last_updated': data['regularMarketTime']} def rate_limit_errmsg(self, elapsed): @@ -752,6 +762,19 @@ def make_cfg(gcfg_arg): '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') else (proxy or cfg_in.cfg.get(name))) + def get_sort_opt(): + match get_cfg_var('sort'): + case None: + return None + case s if s in sort_params: + return (s, True) + case s if s in ['r' + ch for ch in sort_params]: + return (s[1], False) + case s: + die(1, + f'{s!r}: invalid parameter for --sort option (must be one of {fmt_list(sort_params)})' + '\nTo reverse the sort, prefix the code letter with ‘r’') + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -766,6 +789,7 @@ def make_cfg(gcfg_arg): 'proxy', 'proxy2', 'portfolio', + 'sort', 'percent_cols', 'asset_limit', 'cached_data', @@ -823,6 +847,7 @@ def make_cfg(gcfg_arg): proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = portfolio, + sort = get_sort_opt(), percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), @@ -869,6 +894,8 @@ class Ticker: def __init__(self, data): + global cfg + self.comma = ',' if cfg.thousands_comma else '' self.col1_wid = max(len('TOTAL'), ( @@ -885,6 +912,18 @@ class Ticker: for row in rows: self.max_rank = max(self.max_rank, int(data[row.id]['rank'])) + if cfg.sort: + code, reverse = cfg.sort + key = sort_params[code].key + sort_func = lambda row: data[row.id][key] + pf_sort_func = lambda row: data[row[0]][key] + for group in self.rows.keys(): + if group not in self.hidden_groups: + self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) + if cfg.portfolio: + cfg = cfg._replace( + portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -951,6 +990,10 @@ class Ticker: yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) + if cfg.sort: + text = sort_params[cfg.sort[0]].desc + ('' if cfg.sort[1] else ' [reversed]') + yield f'Sort order: {pink(text.upper())}' + for asset in self.usr_col_assets: if asset.symbol != 'USD': usdprice = self.data[asset.id]['price_usd'] diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 921f776..1de14a5 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev6 +3.6.dev7 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index e533e62..7be5e90 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -57,6 +57,9 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. +-s, --sort=P Sort output according to parameter P. Valid parameters + are {sp_codes}. See SORT PARAMETERS below. + To reverse the sort, prefix the parameter with ‘r’. -t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column @@ -133,6 +136,10 @@ A TRADE_SPECIFIER is a single argument in the format: a USD rate for the missing asset(s) must be supplied via the --add-columns or --add-rows options. +SORT PARAMETERS: + + {sp_fmt} + PROXY NOTE @@ -225,6 +232,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, + sp_codes = fmt_list(sort_params, fmt='fancy'), pc = fmt_list(Ticker.percent_cols, fmt='fancy')), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), @@ -232,6 +240,7 @@ To add a portfolio, edit the file pf_cfg = os.path.relpath(cfg_in.portfolio_file, start=homedir), al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), + sp_fmt = '\n '.join(f'‘{k}’ - {v.desc}' for k, v in sort_params.items()), fi = src_cls['fi']()) } } @@ -246,7 +255,7 @@ gcfg = Config(opts_data=opts_data, caller_post_init=True) src_cls, cfg_in = Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, sort_params gcfg._post_init() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 3e909f4..275117f 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -90,6 +90,14 @@ class CmdTestScripts(CmdTestBase): ('ticker19', 'ticker [--cached-data 1-5]'), ('ticker20', 'ticker [--cached-data 2-5]'), ('ticker21', 'ticker [--cached-data 5-5]'), + ('ticker22', 'ticker [--sort=rp]'), + ('ticker23', 'ticker [--sort=rp xmr:10]'), + ('ticker24', 'ticker [--sort=p]'), + ('ticker25', 'ticker [--sort=p 200]'), + ('ticker26', 'ticker [--sort=c -r algo,ada]'), + ('ticker27', 'ticker [--sort=rp -r algo,ada]'), + ('ticker28', 'ticker [--sort=d -r algo,ada]'), + ('ticker29', 'ticker [--sort=y -r algo,ada]'), ) } @@ -365,3 +373,58 @@ class CmdTestScripts(CmdTestBase): r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) + + def ticker22(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], + add_opts = ['--name-labels', '--sort=rp']) + + def ticker23(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], + add_opts = ['--name-labels', '--sort=rp', 'xmr:10']) + + def ticker24(self): + return self.ticker( + [], + ['BITCOIN', 'ETHEREUM', 'MONERO', 'GOLD', 'BRENT', 'SILVER'], + add_opts = ['--name-labels', '--sort=p']) + + def ticker25(self): + return self.ticker( + [], + [ + r' 1\) BITCOIN', + r' 2\) ETHEREUM', + r'30\) MONERO', + r'23\) LITECOIN', + r' 8\) CARDANO', + r'33\) ALGORAND' + ], + add_opts = ['--name-labels', '--sort=p', '200']) + + def ticker26(self): + return self.ticker( + [], + ['BITCOIN', 'ETHEREUM', 'MONERO', 'CARDANO', 'ALGORAND'], + add_opts = ['--name-labels', '--sort=c', '-r', 'ada,algo']) + + def ticker27(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'S&P', 'NASDAQ', 'DOW', 'ALGORAND', 'CARDANO'], + add_opts = ['--name-labels', '--sort=rp', '--add-rows=ada-cardano,algo-algorand']) + + def ticker28(self): + return self.ticker( + [], + ['ETHEREUM', 'MONERO', 'BITCOIN', 'NASDAQ', 'S&P', 'DOW', 'CARDANO', 'ALGORAND'], + add_opts = ['--widest', '--sort=d', '-r', 'ada,algo']) + + def ticker29(self): + return self.ticker( + [], + ['ETHEREUM', 'BITCOIN', 'MONERO', 'S&P', 'DOW', 'NASDAQ', 'CARDANO', 'ALGORAND'], + add_opts = ['--widest', '-s', 'y', '-r', 'ada,algo']) From 13234e990bc70b2cd53b78faaed121fc584f8cd8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 17 Oct 2025 09:07:55 +0000 Subject: [PATCH 73/81] cmdtest.py misc.ticker: cleanups --- test/cmdtest_d/misc.py | 43 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 275117f..9060731 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -67,7 +67,7 @@ class CmdTestScripts(CmdTestBase): 'ticker': ( "'mmnode-ticker' script", ('ticker1', 'ticker [--help]'), - ('copy_files', 'copying JSON files to cache'), + ('copy_cache_files', 'copying JSON files to cache'), ('ticker1a', 'ticker [--download=cc] (early caching)'), ('ticker1b', 'ticker [--download=cc] (late caching)'), ('ticker2', 'ticker (bad proxy)'), @@ -106,17 +106,24 @@ class CmdTestScripts(CmdTestBase): return self.ticker_server = TickerServer(cfg) self.ticker_server.start() + self.dests = { + 'nt_datadir': os.path.join(cfg.data_dir_root, 'node_tools'), + 'cache': self.tmpdir} return super().__init__(cfg, trunner, cfgs, spawn) - @property - def nt_datadir(self): - return os.path.join( cfg.data_dir_root, 'node_tools' ) + def rm_file(self, fn, dest='nt_datadir'): + os.unlink(os.path.join(self.dests[dest], fn)) - def copy_files(self): - self.spawn('',msg_only=True) - shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) - shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) - shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) + def copy_file(self, src_fn, dest_fn=None, dest='nt_datadir'): + shutil.copy2( + os.path.join(refdir, src_fn), + os.path.join(self.dests[dest], dest_fn or src_fn)) + + def copy_cache_files(self): + self.spawn('', msg_only=True) + self.copy_file('ticker-finance.json', dest='cache') + self.copy_file('ticker-finance-history.json', dest='cache') + self.copy_file('ticker-btc.json', dest='cache') return 'ok' def ticker( @@ -186,7 +193,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker5(self): - shutil.copy2(os.path.join(refdir,'ticker-cfg.yaml'),self.nt_datadir) + self.copy_file('ticker-cfg.yaml') t = self.ticker( ['--wide','--adjust=-0.766'], [ @@ -195,7 +202,7 @@ class CmdTestScripts(CmdTestBase): r'LITECOIN 58.56 0.00251869 \+12.79 \+0.40 2022-08-02 18:25:59', r'MONERO 157.76 0.00678495 \+7.28 \+1.21' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-cfg.yaml')) + self.rm_file('ticker-cfg.yaml') return t def ticker6(self): @@ -204,7 +211,7 @@ class CmdTestScripts(CmdTestBase): return t def ticker7(self): # demo - shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + self.copy_file('ticker-portfolio.yaml') t = self.ticker( ['--wide','--portfolio'], [ @@ -213,7 +220,7 @@ class CmdTestScripts(CmdTestBase): 'CARDANO','ALGORAND', 'PORTFOLIO','BITCOIN','ETHEREUM','MONERO','CARDANO','ALGORAND','TOTAL' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker8(self): @@ -225,9 +232,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker9(self): - shutil.copy2( - os.path.join(refdir,'ticker-portfolio-bad.yaml'), - os.path.join(self.nt_datadir,'ticker-portfolio.yaml') ) + self.copy_file('ticker-portfolio-bad.yaml', 'ticker-portfolio.yaml') t = self.ticker( ['--wide','--portfolio','--elapsed','--add-rows=fake-fakecoin:0.0123','--add-precision=2'], [ @@ -236,7 +241,7 @@ class CmdTestScripts(CmdTestBase): r'FAKECOIN 81.3008 0.0034966927 -- -- --', r'\(no data for noc-nocoin\)', ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker10(self): @@ -287,7 +292,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker14(self): - shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + self.copy_file('ticker-portfolio.yaml') t = self.ticker( ['--btc','--wide','--portfolio','--elapsed'], [ @@ -296,7 +301,7 @@ class CmdTestScripts(CmdTestBase): 'PORTFOLIO', r'BITCOIN 28,850.44 \+6.05 -1.87 1.23456789' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker15(self): From 8f9c4ba48ccf695b48d718c6d95a8b717ef85a4c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 17 Oct 2025 09:08:00 +0000 Subject: [PATCH 74/81] mmnode-ticker --sort: handle missing entries in source data --- mmgen_node_tools/Ticker.py | 19 ++++++++++--------- mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 5 ++++- test/ref/ticker/ticker-cfg-bad.yaml | 11 +++++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-bad.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index d3045d4..925a4f9 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -41,14 +41,14 @@ percent_cols = { 'm': 'month', 'y': 'year'} -sp = namedtuple('sort_parameter', ['key', 'desc']) +sp = namedtuple('sort_parameter', ['key', 'sort_dfl', 'desc']) sort_params = { - 'd': sp('percent_change_24h', '1-day % change'), - 'w': sp('percent_change_7d', '1-week % change'), - 'm': sp('percent_change_30d', '1-month % change'), - 'y': sp('percent_change_1y', '1-year % change'), - 'p': sp('price_usd', 'asset price'), - 'c': sp('market_cap', 'market cap')} + 'd': sp('percent_change_24h', 0.0, '1-day % change'), + 'w': sp('percent_change_7d', 0.0, '1-week % change'), + 'm': sp('percent_change_30d', 0.0, '1-month % change'), + 'y': sp('percent_change_1y', 0.0, '1-year % change'), + 'p': sp('price_usd', Decimal(0), 'asset price'), + 'c': sp('market_cap', 0, 'market cap')} class RowDict(dict): @@ -915,8 +915,9 @@ class Ticker: if cfg.sort: code, reverse = cfg.sort key = sort_params[code].key - sort_func = lambda row: data[row.id][key] - pf_sort_func = lambda row: data[row[0]][key] + sort_dfl = sort_params[code].sort_dfl + sort_func = lambda row: data.get(row.id, {key: sort_dfl})[key] + pf_sort_func = lambda row: data.get(row[0], {key: sort_dfl})[key] for group in self.rows.keys(): if group not in self.hidden_groups: self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1de14a5..6eb528d 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev7 +3.6.dev8 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9060731..a3edd34 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -380,10 +380,13 @@ class CmdTestScripts(CmdTestBase): add_opts = ['--add-columns=eurusd=x']) def ticker22(self): - return self.ticker( + self.copy_file('ticker-cfg-bad.yaml', 'ticker-cfg.yaml') + t = self.ticker( [], ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], add_opts = ['--name-labels', '--sort=rp']) + self.rm_file('ticker-cfg.yaml') + return t def ticker23(self): return self.ticker( diff --git a/test/ref/ticker/ticker-cfg-bad.yaml b/test/ref/ticker/ticker-cfg-bad.yaml new file mode 100644 index 0000000..da670c6 --- /dev/null +++ b/test/ref/ticker/ticker-cfg-bad.yaml @@ -0,0 +1,11 @@ +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + - bad-badcoin + commodity: + - gc=f + - si=f + - bz=f From 2647fa1fe3d511bec6ca841101c8e1472f479551 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Oct 2025 09:14:30 +0000 Subject: [PATCH 75/81] mmnode-ticker: minor cleanups --- mmgen_node_tools/Ticker.py | 30 ++++++++++++++++-------------- mmgen_node_tools/main_ticker.py | 4 ++-- test/cmdtest_d/misc.py | 2 ++ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 925a4f9..7bcb567 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -43,10 +43,10 @@ percent_cols = { sp = namedtuple('sort_parameter', ['key', 'sort_dfl', 'desc']) sort_params = { - 'd': sp('percent_change_24h', 0.0, '1-day % change'), - 'w': sp('percent_change_7d', 0.0, '1-week % change'), - 'm': sp('percent_change_30d', 0.0, '1-month % change'), - 'y': sp('percent_change_1y', 0.0, '1-year % change'), + 'd': sp('percent_change_24h', 0.0, '1-day percent change'), + 'w': sp('percent_change_7d', 0.0, '1-week percent change'), + 'm': sp('percent_change_30d', 0.0, '1-month percent change'), + 'y': sp('percent_change_1y', 0.0, '1-year percent change'), 'p': sp('price_usd', Decimal(0), 'asset price'), 'c': sp('market_cap', 0, 'market cap')} @@ -1025,8 +1025,8 @@ class Ticker: if cfg.asset_range: yield from process_rows(self.rows['asset_list']) else: - for rows in self.rows.values(): - if rows: + for group, rows in self.rows.items(): + if rows and group not in self.hidden_groups: yield from process_rows(rows) yield '-' * self.hl_wid @@ -1082,8 +1082,10 @@ class Ticker: def fmt_row(self, d, amt=None, amt_fmt=None): - def fmt_pct(n): - return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') + def fmt_pct(d, key, wid=7): + if (n := d.get(key)) is None: + return gray(' --') + return (red, green)[n>=0](f'{n:+{wid}.2f}') p = self.prices[d['id']] @@ -1096,10 +1098,10 @@ class Ticker: idx = int(d['rank']) if cfg.asset_range else None, mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], - pc1 = fmt_pct(d.get('percent_change_7d')), - pc2 = fmt_pct(d.get('percent_change_24h')), - pc3 = fmt_pct(d.get('percent_change_1y')), - pc4 = fmt_pct(d.get('percent_change_30d')), + pc1 = fmt_pct(d, 'percent_change_7d'), + pc2 = fmt_pct(d, 'percent_change_24h'), + pc3 = fmt_pct(d, 'percent_change_1y', wid=8), + pc4 = fmt_pct(d, 'percent_change_30d'), upd = d.get('last_updated_fmt'), amt = amt_fmt, **{k.replace('-', '_'): v * (1 if amt is None else amt) for k, v in p.items()}) @@ -1117,7 +1119,7 @@ class Ticker: col_fs_data = { 'label': fd(f'{{lbl:{self.col1_wid}}}', f'{{lbl:{self.col1_wid}}}', self.col1_wid), - 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), + 'pct1y': fd(' {pc3:8}', ' {pc3:8}', 9), 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), @@ -1169,7 +1171,7 @@ class Ticker: mcap = 'MarketCap(B)', pc1 = ' CHG_7d', pc2 = 'CHG_24h', - pc3 = 'CHG_1y', + pc3 = ' CHG_1y', pc4 = 'CHG_30d', upd = 'UPDATED', amt = ' AMOUNT', diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 7be5e90..d4c0055 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -40,10 +40,10 @@ opts_data = { used to supply a USD exchange rate for missing assets. -C, --cached-data Use cached data from previous network query instead of live data from server --D, --cachedir=D Read and write cached JSON data to directory ‘D’ - instead of ‘~/{dfl_cachedir}’ -d, --download=D Retrieve and cache asset data ‘D’ from network (valid options: {ds}) +-D, --cachedir=D Read and write cached JSON data to directory ‘D’ + instead of ‘~/{dfl_cachedir}’ -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index a3edd34..c2a4188 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -133,6 +133,7 @@ class CmdTestScripts(CmdTestBase): cached_data = True, add_opts = [], use_proxy = True, + no_msg = False, exit_val = None): t = self.spawn( 'mmnode-ticker', @@ -141,6 +142,7 @@ class CmdTestScripts(CmdTestBase): + (['--proxy=http://asdfzxcv:32459'] if use_proxy else []) + add_opts + args, + no_msg = no_msg, exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) From 84e8ea65d0b86cab2896125e1f1022d4848742a8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Oct 2025 09:14:34 +0000 Subject: [PATCH 76/81] mmnode-ticker: display percent change columns in terms of non-USD assets Any crypto or finance asset may be specified. Examples: # Display percentage changes in relation to Bitcoin: $ mmnode-ticker --widest --pchg-unit=btc # In relation to Gold: $ mmnode-ticker --widest --pchg-unit=gc=f # In relation to Euros: $ mmnode-ticker --widest --pchg-unit=eurusd=x # In relation to the Nasdaq Index: $ mmnode-ticker --widest --pchg-unit=^ixic --- mmgen_node_tools/Ticker.py | 29 +++++++++++++++-- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 2 ++ test/cmdtest_d/misc.py | 38 +++++++++++++++++++++++ test/ref/ticker/ticker-cfg-sort-pchg.yaml | 14 +++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-sort-pchg.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7bcb567..4b49b44 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -535,6 +535,10 @@ def gen_data(data): 'name': 'US Dollar', 'price_usd': Decimal(1), 'price_btc': Decimal(1) / btcusd, + 'percent_change_24h': 0.0, + 'percent_change_7d': 0.0, + 'percent_change_30d': 0.0, + 'percent_change_1y': 0.0, 'last_updated': None}) def cache_data(data_src, no_overwrite=False): @@ -742,7 +746,8 @@ def make_cfg(gcfg_arg): {k: tuple(parse_asset_id(e) for e in v) for k, v in cfg_in.cfg['assets'].items()}) for hdr, data in ( ('user_uniq', get_usr_assets()), - ('portfolio_uniq', get_portfolio_assets())): + ('portfolio_uniq', get_portfolio_assets()), + ('pchg_unit_uniq', [pchg_unit] if pchg_unit else None)): if data: if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): rows[hdr] = uniq_data @@ -791,6 +796,7 @@ def make_cfg(gcfg_arg): 'portfolio', 'sort', 'percent_cols', + 'pchg_unit', 'asset_limit', 'cached_data', 'elapsed', @@ -833,6 +839,9 @@ def make_cfg(gcfg_arg): if portfolio and asset_range: die(1, '--portfolio not supported in market cap view') + pchg_unit = (lambda s: parse_asset_id(s, require_label=False) if s else None)( + get_cfg_var('pchg_unit')) + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, @@ -849,6 +858,7 @@ def make_cfg(gcfg_arg): portfolio = portfolio, sort = get_sort_opt(), percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), + pchg_unit = pchg_unit, asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), elapsed = get_cfg_var('elapsed'), @@ -890,7 +900,7 @@ class Ticker: offer = None to_asset = None - hidden_groups = ('extra',) + hidden_groups = ('extra', 'pchg_unit_uniq') def __init__(self, data): @@ -925,6 +935,14 @@ class Ticker: cfg = cfg._replace( portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + if cfg.pchg_unit: + self.pchg_data = self.data[self.get_id(cfg.pchg_unit)] + self.pchg_factors = {k: (self.pchg_data[k] / 100) + 1 for k in ( + 'percent_change_24h', + 'percent_change_7d', + 'percent_change_30d', + 'percent_change_1y')} + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -995,6 +1013,11 @@ class Ticker: text = sort_params[cfg.sort[0]].desc + ('' if cfg.sort[1] else ' [reversed]') yield f'Sort order: {pink(text.upper())}' + if cfg.pchg_unit: + yield 'Percent change unit: {}'.format(orange('{} ({})'.format( + self.pchg_data['symbol'], + self.pchg_data['name'].upper()))) + for asset in self.usr_col_assets: if asset.symbol != 'USD': usdprice = self.data[asset.id]['price_usd'] @@ -1085,6 +1108,8 @@ class Ticker: def fmt_pct(d, key, wid=7): if (n := d.get(key)) is None: return gray(' --') + if cfg.pchg_unit: + n = ((((n / 100) + 1) / self.pchg_factors[key]) - 1) * 100 return (red, green)[n>=0](f'{n:+{wid}.2f}') p = self.prices[d['id']] diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6eb528d..47203f7 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev8 +3.6.dev9 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index d4c0055..cfca2ac 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -63,6 +63,8 @@ opts_data = { -t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column +-U, --pchg-unit=A Use asset ‘A’ as unit of reference for percentage + change columns (default: USD) -v, --verbose Be more verbose -w, --wide Display most optional columns (same as -unT -p d,w) -W, --widest Display all optional columns (same as -unT -p d,w,m,y) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index c2a4188..9112f48 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -98,6 +98,9 @@ class CmdTestScripts(CmdTestBase): ('ticker27', 'ticker [--sort=rp -r algo,ada]'), ('ticker28', 'ticker [--sort=d -r algo,ada]'), ('ticker29', 'ticker [--sort=y -r algo,ada]'), + ('ticker30', 'ticker [--cached-data --wide --pchg-unit=btc --sort=d] (cf with config file)'), + ('ticker31', 'ticker [--cached-data --wide --pchg-unit=usd] (cf with no USD)'), + ('ticker32', 'ticker [--cached-data --wide --pchg-unit=gc=f]'), ) } @@ -438,3 +441,38 @@ class CmdTestScripts(CmdTestBase): [], ['ETHEREUM', 'BITCOIN', 'MONERO', 'S&P', 'DOW', 'NASDAQ', 'CARDANO', 'ALGORAND'], add_opts = ['--widest', '-s', 'y', '-r', 'ada,algo']) + + def ticker30(self): + self.copy_file('ticker-cfg-sort-pchg.yaml', 'ticker-cfg.yaml') + t = self.ticker(add_opts=['--wide']) + chk1 = '\n'.join(t.read().splitlines()[5:-2]) + self.rm_file('ticker-cfg.yaml') + + self.copy_file('ticker-cfg-bad.yaml', 'ticker-cfg.yaml') + t = self.ticker(add_opts=['--wide', '--pchg-unit=btc', '--sort=d'], no_msg=True) + chk2 = '\n'.join(t.read().splitlines()[5:-2]) + self.rm_file('ticker-cfg.yaml') + + assert chk1 == chk2, f'\nOUTPUT 1\n{chk1}\n!= OUTPUT 2\n{chk2}\n' + return t + + def ticker31(self): + t = self.ticker(add_opts=['--wide']) + chk1 = '\n'.join(t.read().splitlines()[5:-2]) + + t = self.ticker(add_opts=['--wide', '--pchg-unit=usd'], no_msg=True) + chk2 = '\n'.join(t.read().splitlines()[6:-2]) + + assert chk1 == chk2, f'\nOUTPUT 1\n{chk1}\n!= OUTPUT 2\n{chk2}\n' + return t + + def ticker32(self): + return self.ticker( + [], + [ + 'BITCOIN', r'\+10.99', r'\+7.06', '-1.18', r'\+1.05', + 'ETHEREUM', + 'GOLD', r'\+0.00', r'\+0.00', r'\+0.00', r'\+0.00', + 'SILVER' + ], + add_opts = ['--widest', '--pchg-unit=gc=f']) diff --git a/test/ref/ticker/ticker-cfg-sort-pchg.yaml b/test/ref/ticker/ticker-cfg-sort-pchg.yaml new file mode 100644 index 0000000..4660ae4 --- /dev/null +++ b/test/ref/ticker/ticker-cfg-sort-pchg.yaml @@ -0,0 +1,14 @@ +sort: d +pchg_unit: btc + +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + - bad-badcoin + commodity: + - gc=f + - si=f + - bz=f From d02c9936ef7504e70b07e82401042ae5fc58901a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 22 Oct 2025 10:33:16 +0000 Subject: [PATCH 77/81] mmnode-ticker: fix market cap sort with USD row --- mmgen_node_tools/Ticker.py | 1 + mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 26 +++++++++++++++++++++++++- test/ref/ticker/ticker-cfg-usd.yaml | 11 +++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-usd.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 4b49b44..15e0e9c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -539,6 +539,7 @@ def gen_data(data): 'percent_change_7d': 0.0, 'percent_change_30d': 0.0, 'percent_change_1y': 0.0, + 'market_cap': 0, 'last_updated': None}) def cache_data(data_src, no_overwrite=False): diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 47203f7..bd10d35 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev9 +3.6.dev10 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9112f48..d5cb51d 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -101,6 +101,9 @@ class CmdTestScripts(CmdTestBase): ('ticker30', 'ticker [--cached-data --wide --pchg-unit=btc --sort=d] (cf with config file)'), ('ticker31', 'ticker [--cached-data --wide --pchg-unit=usd] (cf with no USD)'), ('ticker32', 'ticker [--cached-data --wide --pchg-unit=gc=f]'), + ('ticker33', 'ticker [--cached-data --wide --pchg-unit=btc --sort=c] (cfg file with USD)'), + ('ticker34', 'ticker [--cached-data --wide --pchg-unit=btc --sort=y] (cfg file with USD)'), + ('ticker35', 'ticker [--cached-data --wide --pchg-unit=btc --sort=p] (cfg file with USD)'), ) } @@ -475,4 +478,25 @@ class CmdTestScripts(CmdTestBase): 'GOLD', r'\+0.00', r'\+0.00', r'\+0.00', r'\+0.00', 'SILVER' ], - add_opts = ['--widest', '--pchg-unit=gc=f']) + add_opts = ['--widest', '--pchg-unit=gc=f', '--sort=c']) + + def _ticker_cur(self, sort): + self.copy_file('ticker-cfg-usd.yaml', 'ticker-cfg.yaml') + t = self.ticker( + [], + [ + 'BITCOIN 23,250.77 1.00000000 \+0.00', + 'US DOLLAR 1.00 0.00004301 -15.93', + ], + add_opts = ['--widest', '--pchg-unit=btc', f'--sort={sort}']) + self.rm_file('ticker-cfg.yaml') + return t + + def ticker33(self): + return self._ticker_cur(sort='c') + + def ticker34(self): + return self._ticker_cur(sort='y') + + def ticker35(self): + return self._ticker_cur(sort='p') diff --git a/test/ref/ticker/ticker-cfg-usd.yaml b/test/ref/ticker/ticker-cfg-usd.yaml new file mode 100644 index 0000000..c31f85f --- /dev/null +++ b/test/ref/ticker/ticker-cfg-usd.yaml @@ -0,0 +1,11 @@ +assets: + coin1: + - btc-bitcoin + - eth-ethereum + - xmr-monero + commodity: + - gc=f + - si=f + currency: + - usd-us-dollar + - eurusd=x From f5c165c36201c0b251298fd08cef9566c42b37ee Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:31 +0000 Subject: [PATCH 78/81] whitespace, minor cleanups and fixes --- mmgen_node_tools/BlocksInfo.py | 5 +- mmgen_node_tools/Ticker.py | 3 +- test/cmdtest_d/regtest.py | 110 +++++++++++++++++---------------- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index a59bf50..c068017 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -353,7 +353,8 @@ class BlocksInfo: nblocks, step = (add1, add2) if last is None else (None, add1) - if p.debug: msg(repr(self.range_data(first, last, from_tip, nblocks, step))) + if p.debug: + msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if nblocks: if first is None: @@ -643,9 +644,7 @@ class BlocksInfo: values = {n: d.func(n) for n in fnames} col1_w = max((len(l) for l in lbls.values()), default=0) + 2 - print(d.spec_vals) for v in d.spec_vals: - print(v) if v.condition(values): try: idx = fnames.index(v.insert_after) + 1 except: idx = 0 diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 15e0e9c..50bb85b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -970,8 +970,7 @@ class Ticker: t = int(d[row.id]['last_updated']) except TypeError as e: d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) - except KeyError as e: - msg(str(e)) + except KeyError: pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index a504678..b35ec7a 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -23,18 +23,18 @@ from ..include.common import cfg, imsg, stop_test_daemons, joinpath from .base import CmdTestBase args1 = ['--bob'] -args2 = ['--bob','--rpc-backend=http'] +args2 = ['--bob', '--rpc-backend=http'] -def gen_addrs(proto,network,keys): +def gen_addrs(proto, network, keys): from mmgen.tool.api import tool_api tool = tool_api(cfg) - tool.init_coin(proto.coin,'regtest') + tool.init_coin(proto.coin, 'regtest') tool.addrtype = proto.mmtypes[-1] return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' - networks = ('btc','ltc','bch') + networks = ('btc', 'ltc', 'bch') passthru_opts = ('coin',) tmpdir_nums = [1] color = True @@ -110,31 +110,31 @@ class CmdTestRegtest(CmdTestBase): if trunner is None: return if cfg._proto.testnet: - die(2,'--testnet and --regtest options incompatible with regtest test suite') + die(2, '--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) - self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto,'regtest',[1,2,3,4,5])] + self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto, 'regtest', [1, 2, 3, 4, 5])] self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): - stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) + stop_test_daemons(self.proto.network_id, force=True, remove_datadir=True) from shutil import rmtree try: - rmtree(joinpath(self.tr.data_dir,'regtest')) + rmtree(joinpath(self.tr.data_dir, 'regtest')) except: pass t = self.spawn( 'mmgen-regtest', (['--bdb-wallet'] if self.use_bdb_wallet else []) + ['--setup-no-stop-daemon', 'setup']) - for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): + for s in ('Starting', 'Creating', 'Creating', 'Creating', 'Mined', 'Setup complete'): t.expect(s) return t def netrate(self, add_args, expect_str, exit_val=None): t = self.spawn('mmnode-netrate', args1 + add_args, exit_val=exit_val) - t.expect(expect_str,regex=True) + t.expect(expect_str, regex=True) return t def netrate1(self): @@ -147,35 +147,35 @@ class CmdTestRegtest(CmdTestBase): return 'ok' return t - def halving_calculator(self,add_args,expect_list): - t = self.spawn('mmnode-halving-calculator',args1+add_args) + def halving_calculator(self, add_args, expect_list): + t = self.spawn('mmnode-halving-calculator', args1+add_args) t.match_expect_list(expect_list) return t def halving_calculator1(self): - return self.halving_calculator(['--help'],['USAGE:']) + return self.halving_calculator(['--help'], ['USAGE:']) def halving_calculator2(self): - return self.halving_calculator([],['Current block: 393',f'Current block subsidy: 12.5 {cfg.coin}']) + return self.halving_calculator([], ['Current block: 393', f'Current block subsidy: 12.5 {cfg.coin}']) def halving_calculator3(self): - return self.halving_calculator(['--list'],['33 4950','0']) + return self.halving_calculator(['--list'], ['33 4950', '0']) def halving_calculator4(self): - return self.halving_calculator(['--mined'],['0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined'], ['0 0.0000015 14949.9999835']) def halving_calculator5(self): - return self.halving_calculator(['--mined','--bdr-proj=5'],['5.00000 0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined', '--bdr-proj=5'], ['5.00000 0 0.0000015 14949.9999835']) def halving_calculator6(self): - return self.halving_calculator(['--mined','--sample-size=20'],['33 4950','0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined', '--sample-size=20'], ['33 4950', '0 0.0000015 14949.9999835']) - def sendto(self,addr,amt): - return self.spawn('mmgen-regtest',['send',addr,amt]) + def sendto(self, addr, amt): + return self.spawn('mmgen-regtest', ['send', addr, amt]) - def sendto1(self): return self.sendto(self.addrs[0],'0.123') - def sendto2(self): return self.sendto(self.addrs[0],'0.234') - def sendto3(self): return self.sendto(self.addrs[1],'0.345') + def sendto1(self): return self.sendto(self.addrs[0], '0.123') + def sendto2(self): return self.sendto(self.addrs[0], '0.234') + def sendto3(self): return self.sendto(self.addrs[1], '0.345') def addrbal(self, args, expect_list): t = self.spawn('mmnode-addrbal', args2 + args) @@ -231,7 +231,7 @@ class CmdTestRegtest(CmdTestBase): [ 'No balance', '2 unspent outputs in 2 blocks', - '394','0.123','395','0.234', + '394', '0.123', '395', '0.234', 'No balance' ]) @@ -240,7 +240,7 @@ class CmdTestRegtest(CmdTestBase): ['--tabular', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - -', - self.addrs[0] + ' 2 395','0.357', + self.addrs[0] + ' 2 395', '0.357', self.addrs[3] + ' - - -', ]) @@ -249,11 +249,11 @@ class CmdTestRegtest(CmdTestBase): ['--tabular', '--first-block', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - - -', - self.addrs[0] + ' 2 394','395','0.357', + self.addrs[0] + ' 2 394', '395', '0.357', self.addrs[3] + ' - - - -', ]) - def blocks_info(self,args,expect_list): + def blocks_info(self, args, expect_list): t = self.spawn('mmnode-blocks-info', args1 + args) t.match_expect_list(expect_list) return t @@ -261,7 +261,7 @@ class CmdTestRegtest(CmdTestBase): def blocks_info1(self): return self.blocks_info( ['--help'], - ['USAGE:','OPTIONS:']) + ['USAGE:', 'OPTIONS:']) def blocks_info2(self): return self.blocks_info( @@ -278,7 +278,7 @@ class CmdTestRegtest(CmdTestBase): ]) def blocks_info4(self): - n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) + n1, i1, o1, n2, i2, o2 = (2, 1, 3, 6, 3, 9) if cfg.coin == 'BCH' else (2, 1, 4, 6, 3, 12) return self.blocks_info( ['--miner-info', '--fields=all', '--stats=all', '+3'], [ @@ -302,21 +302,21 @@ class CmdTestRegtest(CmdTestBase): from collections import namedtuple t = tool_api(cfg) - t.init_coin(self.proto.coin,self.proto.network) + t.init_coin(self.proto.coin, self.proto.network) t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32' - wp = namedtuple('wifaddrpair',['wif','addr']) + wp = namedtuple('wifaddrpair', ['wif', 'addr']) def gen(): - for n in range(0xfaceface,nPairs+0xfaceface): + for n in range(0xfaceface, nPairs+0xfaceface): wif = t.hex2wif(f'{n:064x}') yield wp( wif, t.wif2addr(wif) ) return list(gen()) - def gen_fees(n_in,low,high): + def gen_fees(n_in, low, high): # very approximate tx size estimation: - ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) + ibytes, wbytes, obytes = (148, 0, 34) if self.proto.coin == 'BCH' else (43, 108, 31) x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt.satoshi n = n_in - 1 @@ -325,14 +325,20 @@ class CmdTestRegtest(CmdTestBase): for i in range(n_in): yield Decimal(low + (i/n)**6 * vmax) * x - async def do_tx(inputs,outputs,wif): - tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) - tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) + async def do_tx(inputs, outputs, wif): + tx_hex = await r.rpc_call('createrawtransaction', inputs, outputs) + if wif: + tx = await r.rpc_call( + 'signrawtransactionwithkey', + tx_hex, + [wif], + [], + self.proto.sighash_type) assert tx['complete'] return tx['hex'] async def do_tx1(): - us = await r.rpc_call('listunspent',wallet='miner') + us = await r.rpc_call('listunspent', wallet='miner') tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr: tx1_amt for p in pairs[:nTxs]} @@ -342,7 +348,7 @@ class CmdTestRegtest(CmdTestBase): outputs, await r.miner_wif) - async def do_tx2(tx,pairno): + async def do_tx2(tx, pairno): fee = self.proto.coin_amt(fees[pairno], from_decimal=True) outputs = {p.addr: tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) @@ -353,10 +359,10 @@ class CmdTestRegtest(CmdTestBase): async def do_txs(tx_in): for pairno in range(nTxs): - tx_hex = await do_tx2(tx_in,pairno) - await r.rpc_call('sendrawtransaction',tx_hex) + tx_hex = await do_tx2(tx_in, pairno) + await r.rpc_call('sendrawtransaction', tx_hex) - self.spawn('',msg_only=True) + self.spawn('', msg_only=True) r = self.regtest nPairs = 100 @@ -372,20 +378,20 @@ class CmdTestRegtest(CmdTestBase): tx1_hex = await do_tx1() imsg('Relaying funding transaction') - await r.rpc_call('sendrawtransaction',tx1_hex) + await r.rpc_call('sendrawtransaction', tx1_hex) imsg('Mining a block') - await r.generate(1,silent=True) + await r.generate(1, silent=True) imsg('Generating fees for mempool transactions') - fees = list(gen_fees(nTxs,2,120)) + fees = list(gen_fees(nTxs, 2, 120)) imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') - await do_txs(await r.rpc_call('decoderawtransaction',tx1_hex)) + await do_txs(await r.rpc_call('decoderawtransaction', tx1_hex)) return 'ok' - def _feeview(self,args,expect_list=[]): + def _feeview(self, args, expect_list=[]): t = self.spawn('mmnode-feeview', args1 + args) if expect_list: t.match_expect_list(expect_list) @@ -395,7 +401,7 @@ class CmdTestRegtest(CmdTestBase): return self._feeview([]) def feeview2(self): - return self._feeview(['--columns=40','--include-current']) + return self._feeview(['--columns=40', '--include-current']) def feeview3(self): return self._feeview(['--precision=6']) @@ -404,7 +410,7 @@ class CmdTestRegtest(CmdTestBase): return self._feeview(['--detail']) def feeview5(self): - return self._feeview(['--show-empty','--log',f'--outdir={self.tmpdir}']) + return self._feeview(['--show-empty', '--log', f'--outdir={self.tmpdir}']) def feeview6(self): return self._feeview(['--ignore-below=1MB']) @@ -414,13 +420,13 @@ class CmdTestRegtest(CmdTestBase): async def feeview8(self): imsg('Clearing mempool') - await self.regtest.generate(1,silent=True) + await self.regtest.generate(1, silent=True) return self._feeview([]) def stop(self): if cfg.no_daemon_stop: - self.spawn('',msg_only=True) + self.spawn('', msg_only=True) msg_r('(leaving daemon running by user request)') return 'ok' else: - return self.spawn('mmgen-regtest',['stop']) + return self.spawn('mmgen-regtest', ['stop']) From e5bc33a6d6267ec320ddf4ae3511fc4590e4ec1b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:32 +0000 Subject: [PATCH 79/81] update for MMGen Wallet v16.1.dev21 --- mmgen_node_tools/main_addrbal.py | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 4647029..743b36e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -44,7 +44,7 @@ def do_output(proto, addr_data, blk_hdrs): heights = {u['height'] for u in unspents} Msg('{}Balance: {}'.format( indent, - sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), + sum(proto.coin_amt(u['amount']) for u in unspents).hl3(unit=True, fs='{:,}'))), Msg('{}{} unspent output{} in {} block{}'.format( indent, red(str(len(unspents))), @@ -148,7 +148,7 @@ async def main(req_addrs): good_addrs = len([v for v in addr_data.values() if v]) Msg('Total: {} in {} address{}'.format( - proto.coin_amt(res['total_amount']).hl2(unit=True, fs='{:,}'), + proto.coin_amt(res['total_amount']).hl3(unit=True, fs='{:,}'), red(str(good_addrs)), suf(good_addrs, 'es'))) diff --git a/setup.cfg b/setup.cfg index d48a8c8..91b63b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev4 + mmgen-wallet>=16.1.dev21 pyyaml yahooquery From b97b4f5f634f2498947f4ea4b25f5ac71d08db85 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:32 +0000 Subject: [PATCH 80/81] update for Bitcoin Core v30.0, nixpkgs 25.11 --- mmgen_node_tools/data/version | 2 +- nix/curl-cffi.nix | 50 ----------------------------------- nix/yahooquery.nix | 2 +- test/cmdtest_d/regtest.py | 9 ++++++- 4 files changed, 10 insertions(+), 53 deletions(-) delete mode 100644 nix/curl-cffi.nix diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index bd10d35..4e5cab0 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev10 +3.6.dev11 diff --git a/nix/curl-cffi.nix b/nix/curl-cffi.nix deleted file mode 100644 index 15f9764..0000000 --- a/nix/curl-cffi.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ - stdenv, - lib, - python, - buildPythonPackage, - fetchPypi, - curl-impersonate-chrome, -}: - -buildPythonPackage rec { - pname = "curl-cffi"; - # version = "0.13.0"; # uses option PROXY_CREDENTIAL_NO_REUSE, unavailable in current libcurl - version = "0.10.0"; - pyproject = true; - - src = fetchPypi { - pname = "curl_cffi"; - version = version; - # hash = "sha256-YuzZCjgr1QI3UONgbgqnyxo6i6QcFCcLjl4Unr9yxco="; # 0.13.0 - hash = "sha256-PjezUmjKWEkvVO0CCuS1DDPuDeutQUXbn3RvBO1GbrA="; # 0.10.0 - }; - - patches = [ ./use-system-libs.patch ]; - - buildInputs = [ curl-impersonate-chrome ]; - - build-system = with python.pkgs; [ - cffi - setuptools - ]; - - dependencies = with python.pkgs; [ - cffi - certifi - typing-extensions - ]; - - env = lib.optionalAttrs stdenv.cc.isGNU { - NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types"; - }; - - pythonImportsCheck = [ "curl_cffi" ]; - - meta = with lib; { - description = "Python binding for curl-impersonate via cffi"; - homepage = "https://curl-cffi.readthedocs.io"; - license = licenses.mit; - maintainers = with maintainers; [ chuangzhu ]; - }; -} diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix index 3e8925f..ecb2494 100644 --- a/nix/yahooquery.nix +++ b/nix/yahooquery.nix @@ -19,7 +19,7 @@ buildPythonPackage rec { build-system = with python.pkgs; [ hatchling ]; propagatedBuildInputs = with python.pkgs; [ - (callPackage ./curl-cffi.nix {}) # >=0.10.0 + curl-cffi pandas requests-futures tqdm diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index b35ec7a..387594b 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -114,7 +114,7 @@ class CmdTestRegtest(CmdTestBase): self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto, 'regtest', [1, 2, 3, 4, 5])] - self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' + self.use_bdb_wallet = self.bdb_wallet and self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): @@ -334,6 +334,13 @@ class CmdTestRegtest(CmdTestBase): [wif], [], self.proto.sighash_type) + else: + tx = await r.rpc_call( + 'signrawtransactionwithwallet', + tx_hex, + None, # prevtxs + self.proto.sighash_type, + wallet = 'miner') assert tx['complete'] return tx['hex'] From f88b70c2e2413d6261dcc3d5d5d930ec778d9eff Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 27 Jan 2026 08:46:15 +0000 Subject: [PATCH 81/81] mmnode-ticker: support nested assets in `ticker-portfolio.yaml` --- mmgen_node_tools/Ticker.py | 39 ++++++++++++++------- mmgen_node_tools/data/ticker-portfolio.yaml | 10 +++++- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/ref/ticker/ticker-portfolio.yaml | 9 +++-- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 50bb85b..a0adb39 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -720,15 +720,28 @@ def make_cfg(gcfg_arg): + usr_columns) def get_portfolio_assets(): - if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e) for e in cfg_in.portfolio) + if portfolio: + ret = (parse_asset_id(e) for e in portfolio) return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') else: return () - def get_portfolio(): - return tuple((k, Decimal(v)) for k, v in cfg_in.portfolio.items() - if (not gcfg.btc) or k == 'btc-bitcoin') + def parse_portfolio(): + ret = {} + def add(k, v): + if gcfg.btc and k != 'btc-bitcoin': + return + if k in ret: + ret[k] += Decimal(v) + else: + ret[k] = Decimal(v) + for k, v in cfg_in.portfolio.items(): + if isinstance(v, dict): + for k2, v2 in v.items(): + add(k2, v2) + else: + add(k, v) + return ret def parse_add_precision(arg): if not arg: @@ -834,7 +847,7 @@ def make_cfg(gcfg_arg): proxy2 = get_proxy('proxy2') portfolio = ( - get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query + parse_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query else None) if portfolio and asset_range: @@ -928,13 +941,14 @@ class Ticker: key = sort_params[code].key sort_dfl = sort_params[code].sort_dfl sort_func = lambda row: data.get(row.id, {key: sort_dfl})[key] - pf_sort_func = lambda row: data.get(row[0], {key: sort_dfl})[key] + pf_sort_func = lambda row: data.get(row, {key: sort_dfl})[key] for group in self.rows.keys(): if group not in self.hidden_groups: self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) if cfg.portfolio: - cfg = cfg._replace( - portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + cfg = cfg._replace(portfolio = + {k: cfg.portfolio[k] + for k in sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)}) if cfg.pchg_unit: self.pchg_data = self.data[self.get_id(cfg.pchg_unit)] @@ -1061,7 +1075,7 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym, amt in cfg.portfolio: + for sym, amt in cfg.portfolio.items(): try: yield self.fmt_row(self.data[sym], amt=amt) except KeyError: @@ -1086,10 +1100,9 @@ class Ticker: self.format_last_updated_col() if cfg.portfolio: - pf_dict = dict(cfg.portfolio) - self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * pf_dict[row.id] + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] for row in self.rows - if row.id in pf_dict and row.id in data) + if row.id in cfg.portfolio and row.id in data) for col_id in self.col_ids} self.init_prec() diff --git a/mmgen_node_tools/data/ticker-portfolio.yaml b/mmgen_node_tools/data/ticker-portfolio.yaml index fcadd30..b63e626 100644 --- a/mmgen_node_tools/data/ticker-portfolio.yaml +++ b/mmgen_node_tools/data/ticker-portfolio.yaml @@ -3,4 +3,12 @@ # Invoke `mmnode-ticker --list-ids` for a full list of supported asset IDs. btc-bitcoin: '1.23456789' eth-ethereum: '2.3456789012' -xmr-monero: '4.5678901234' +xmr-monero: '4.567890123456' + +# Nested values are supported. Values for each asset will be summed. +wallet2: + btc-bitcoin: '0.12345678' + ltc-litecoin: '1.23456789' + +exchange1: + xmr-monero: '12.345678901234' diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4e5cab0..6b99850 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev11 +3.6.dev12 diff --git a/setup.cfg b/setup.cfg index 91b63b2..0753d4f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev21 + mmgen-wallet>=16.1.dev26 pyyaml yahooquery diff --git a/test/ref/ticker/ticker-portfolio.yaml b/test/ref/ticker/ticker-portfolio.yaml index 332257f..ca0c2b2 100644 --- a/test/ref/ticker/ticker-portfolio.yaml +++ b/test/ref/ticker/ticker-portfolio.yaml @@ -1,5 +1,10 @@ btc-bitcoin: '1.23456789' eth-ethereum: '2.345678901234567890' -xmr-monero: '4.567890123456' +xmr-monero: '3.333390123456' ada-cardano: '123.45678901' -algo-algorand: '234.5678901' + +wallet2: + algo-algorand: '234.5678901' + +exchange1: + xmr-monero: '1.2345'