From e94f036c73abe57b41e1ffbf3f715198249fddda Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:22 +0000 Subject: [PATCH] mmnode-ticker: cleanups --- mmgen_node_tools/Ticker.py | 80 ++++++++++++++++++++------------- mmgen_node_tools/main_ticker.py | 14 +++--- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 1e21192..fc0b8da 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -32,26 +32,34 @@ dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) - -def fetch_delay(fetched_data=[]): - if not gcfg.testing: - if fetched_data: - delay = 1 + random.randrange(1,5000) / 1000 - msg_r(f'Waiting {delay:.3f} seconds...') - time.sleep(delay) - msg('') - else: - fetched_data.append(None) +last_api_host = None class DataSource: - sources = { - 'cc': 'coinpaprika', - 'fi': 'yahooquery' - } + source_groups = [ + { + 'cc': 'coinpaprika' + }, { + 'fi': 'yahoospot', + } + ] + + @classmethod + def get_sources(cls,randomize=False): + g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups + return {k:v for a in g for k,v in a.items()} class base: + def fetch_delay(self): + global last_api_host + if not gcfg.testing and last_api_host and last_api_host != self.api_host: + delay = 1 + random.randrange(1,5000) / 1000 + msg_r(f'Waiting {delay:.3f} seconds...') + time.sleep(delay) + msg('') + last_api_host = self.api_host + def get_data_from_network(self): curl_cmd = list_gen( @@ -90,11 +98,11 @@ class DataSource: else: data_type = self.net_data_type elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) - if elapsed >= self.timeout: + if elapsed >= self.timeout or gcfg.testing: if gcfg.testing: msg('') - fetch_delay() - msg_r(f'Fetching data from {self.api_host}...') + self.fetch_delay() + msg_r(f'Fetching {self.data_desc} from {self.api_host}...') if self.has_verbose: gcfg._util.vmsg('') data_in = self.get_data_from_network() @@ -145,6 +153,7 @@ class DataSource: class coinpaprika(base): desc = 'CoinPaprika' + data_desc = 'cryptocurrency data' api_host = 'api.coinpaprika.com' ratelimit = 240 btc_ratelimit = 10 @@ -201,14 +210,16 @@ class DataSource: id = (s.lower() if label else None), source = 'cc' ) - class yahooquery(base): + class yahoospot(base): desc = 'Yahoo Finance' + data_desc = 'spot financial data' api_host = 'finance.yahoo.com' ratelimit = 30 net_data_type = 'python' has_verbose = False asset_id_pat = r'^\^.*|.*=[xf]$' + json_fn_basename = 'ticker-finance.json' @staticmethod def get_id(sym,data): @@ -233,29 +244,37 @@ class DataSource: @property def json_fn(self): - return os.path.join( cfg.cachedir, 'ticker-finance.json' ) + return os.path.join( cfg.cachedir, self.json_fn_basename ) @property def timeout(self): return 5 if gcfg.test_suite else self.ratelimit + @property + def symbols(self): + return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + def get_data_from_network(self): - arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] - - kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } } + kwargs = { + 'formatted': True, + 'proxies': { 'https': cfg.proxy2 }, + } if gcfg.test_suite: kwargs.update({ 'timeout': 1, 'retry': 0 }) if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( - arg, + self.symbols, fmt_dict(kwargs,fmt='kwargs') )) return from yahooquery import Ticker - return Ticker(arg,**kwargs).price + return self.process_network_data( Ticker(self.symbols,**kwargs) ) + + def process_network_data(self,ticker): + return ticker.price @staticmethod def parse_asset_id(s,require_label): @@ -422,15 +441,13 @@ def main(): if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: - if not gcfg.download in DataSource.sources: + if not gcfg.download in DataSource.get_sources(): die(1,f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: - src_ids = DataSource.sources + src_ids = DataSource.get_sources(randomize=True) - ids = random.sample( list(src_ids), k=len(src_ids) ) # shuffle the ids - - src_data = { k: src_cls[k]().get_data() for k in ids } + src_data = { k: src_cls[k]().get_data() for k in src_ids } if gcfg.testing: return @@ -582,7 +599,7 @@ def make_cfg(gcfg_arg): gcfg = gcfg_arg - src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() } + src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() } fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args @@ -673,8 +690,7 @@ class Ticker: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) # ticker API - # t.replace('T',' ').replace('Z','') # tickers API + fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) d = self.data max_w = 0 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 6a9d3aa..13175b8 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -14,10 +14,10 @@ mmnode-ticker: Display price information for cryptocurrency and other assets opts_data = { 'sets': [ - ('wide', True, 'percent_change', True), - ('wide', True, 'name_labels', True), - ('wide', True, 'thousands_comma', True), - ('wide', True, 'update_time', True), + ('wide', True, 'percent_change', True), + ('wide', True, 'name_labels', True), + ('wide', True, 'thousands_comma', True), + ('wide', True, 'update_time', True), ], 'text': { 'desc': 'Display prices for cryptocurrency and other assets', @@ -35,8 +35,8 @@ opts_data = { live data from server -D, --cachedir=D Read and write cached JSON data to directory ‘D’ instead of ‘~/{dfl_cachedir}’ --d, --download=D Retrieve data ‘D’ from source, save to file and exit - (valid options: {ds}) +-d, --download=D Retrieve and cache asset data ‘D’ from network (valid + options: {ds}) -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data @@ -200,7 +200,7 @@ To add a portfolio, edit the file 'code': { 'options': lambda s: s.format( dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), - ds = fmt_dict(DataSource.sources,fmt='equal'), + ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), ), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '),