mmnode-ticker: support multiple data sources

mmgen_node_tools/Ticker.py

@@ -12,11 +12,6 @@
 mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets
 """
 
-api_host = 'api.coinpaprika.com'
-api_url  = f'https://{api_host}/v1/ticker'
-ratelimit = 240
-btc_ratelimit = 10
-
 # We use deprecated coinpaprika ‘ticker’ API for now because it returns ~45% less data.
 # Old ‘ticker’ API  (/v1/ticker):  data['BTC']['price_usd']
 # New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price']
@@ -24,25 +19,193 @@ btc_ratelimit = 10
 # Possible alternatives:
 # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR
 
-import sys,os,time,json,yaml
+import sys,os,re,time,json,yaml,random
 from subprocess import run,PIPE,CalledProcessError
 from decimal import Decimal
 from collections import namedtuple
 
 from mmgen.color import *
-from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list
+from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen
 
 homedir = os.getenv('HOME')
 dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools')
 cfg_fn = 'ticker-cfg.yaml'
 portfolio_fn = 'ticker-portfolio.yaml'
+asset_tuple = namedtuple('asset_tuple',['symbol','id','source'])
+
+def fetch_delay(fetched_data=[]):
+	if not gcfg.testing:
+		if fetched_data:
+			delay = 1 + random.randrange(1,5000) / 1000
+			msg(f'Waiting {delay:.3f} seconds...')
+			time.sleep(delay)
+		else:
+			fetched_data.append(None)
+
+class DataSource:
+
+	sources = {
+		'cc': 'coinpaprika',
+	}
+
+	class base:
+
+		def get_data_from_network(self):
+
+			curl_cmd = list_gen(
+				['curl', '--tr-encoding', '--header', 'Accept: application/json',True],
+				['--compressed'], # adds 'Accept-Encoding: gzip'
+				['--proxy', cfg.proxy, isinstance(cfg.proxy,str)],
+				['--silent', not gcfg.verbose],
+				[self.api_url]
+			)
+
+			if gcfg.testing:
+				Msg(fmt_list(curl_cmd,fmt='bare'))
+				return
+
+			try:
+				return run(curl_cmd,check=True,stdout=PIPE).stdout.decode()
+			except CalledProcessError as e:
+				msg('')
+				from .Misc import curl_exit_codes
+				msg(red(curl_exit_codes[e.returncode]))
+				msg(red('Command line:\n  {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) )))
+				from mmgen.exception import MMGenCalledProcessError
+				raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}')
+
+		def get_data(self):
+
+			if not os.path.exists(cfg.cachedir):
+				os.makedirs(cfg.cachedir)
+
+			if not os.path.exists(self.json_fn):
+				open(self.json_fn,'w').write('{}')
+
+			if gcfg.cached_data:
+				data_type = 'json'
+				data_in = open(self.json_fn).read()
+			else:
+				data_type = self.net_data_type
+				elapsed = int(time.time() - os.stat(self.json_fn).st_mtime)
+				if elapsed >= self.timeout:
+					if gcfg.testing:
+						msg('')
+					fetch_delay()
+					msg_r(f'Fetching data from {self.api_host}...')
+					if self.has_verbose:
+						gcfg._util.vmsg('')
+					data_in = self.get_data_from_network()
+					msg('done')
+					if gcfg.testing:
+						return {}
+				else:
+					die(1,self.rate_limit_errmsg(elapsed))
+
+			if data_type == 'json':
+				try:
+					data = json.loads(data_in)
+				except:
+					self.json_data_error_msg(data_in)
+					die(2,'Retrieved data is not valid JSON, exiting')
+				json_text = data_in
+			elif data_type == 'python':
+				data = data_in
+				json_text = json.dumps(data_in)
+
+			if not data:
+				if gcfg.cached_data:
+					die(1,'No cached data!  Run command without --cached-data option to retrieve data from remote host')
+				else:
+					die(2,'Remote host returned no data!')
+			elif 'error' in data:
+				die(1,data['error'])
+
+			if gcfg.cached_data:
+				msg(f'Using cached data from ~/{self.json_fn_rel}')
+			else:
+				open(self.json_fn,'w').write(json_text)
+				msg(f'JSON data cached to ~/{self.json_fn_rel}')
+				if gcfg.download:
+					sys.exit(0)
+
+			return self.postprocess_data(data)
+
+		def json_data_error_msg(self,json_text):
+			pass
+
+		def postprocess_data(self,data):
+			return data
+
+		@property
+		def json_fn_rel(self):
+			return os.path.relpath(self.json_fn,start=homedir)
+
+	class coinpaprika(base):
+		desc = 'CoinPaprika'
+		api_host = 'api.coinpaprika.com'
+		ratelimit = 240
+		btc_ratelimit = 10
+		net_data_type = 'json'
+		has_verbose = True
+
+		def rate_limit_errmsg(self,elapsed):
+			return (
+				f'Rate limit exceeded!  Retry in {self.timeout-elapsed} seconds' +
+				('' if cfg.btc_only else ', or use --cached-data or --btc')
+			)
+
+		@property
+		def api_url(self):
+			return f'https://{self.api_host}/v1/ticker' + ('/btc-bitcoin' if cfg.btc_only else '')
+
+		@property
+		def json_fn(self):
+			return os.path.join(
+				cfg.cachedir,
+				'ticker-btc.json' if cfg.btc_only else 'ticker.json' )
+
+		@property
+		def timeout(self):
+			return 5 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit
+
+		def json_data_error_msg(self,json_text):
+			tor_captcha_msg = f"""
+				If you’re using Tor, the API request may have failed due to Captcha protection.
+				A workaround for this issue is to retrieve the JSON data with a browser from
+				the following URL:
+
+					{self.api_url}
+
+				and save it to:
+
+					‘{cfg.cachedir}/ticker.json’
+
+				Then invoke the program with --cached-data and without --btc
+			"""
+			msg(json_text[:1024] + '...')
+			msg(orange(fmt(tor_captcha_msg,strip_char='\t')))
+
+		def postprocess_data(self,data):
+			return [data] if cfg.btc_only else data
+
+		@staticmethod
+		def parse_asset_id(s,require_label):
+			sym,label = (*s.split('-',1),None)[:2]
+			if require_label and not label:
+				die(1,f'{s!r}: asset label is missing')
+			return asset_tuple(
+				symbol = sym.upper(),
+				id     = (s.lower() if label else None),
+				source = 'cc' )
 
 def assets_list_gen(cfg_in):
 	for k,v in cfg_in.cfg['assets'].items():
 		yield('')
 		yield(k.upper())
 		for e in v:
-			yield('  {:4s} {}'.format(*e.split('-',1)))
+			out = e.split('-',1)
+			yield('  {:5s} {}'.format(out[0],out[1] if len(out) == 2 else ''))
 
 def gen_data(data):
 	"""
@@ -59,7 +222,7 @@ def gen_data(data):
 	def dup_sym_errmsg(dup_sym):
 		return (
 			f'The symbol {dup_sym!r} is shared by the following assets:\n' +
-			'\n  ' + '\n  '.join(d['id'] for d in data if d['symbol'] == dup_sym) +
+			'\n  ' + '\n  '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) +
 			'\n\nPlease specify the asset by one of the full IDs listed above\n' +
 			f'instead of {dup_sym!r}'
 		)
@@ -103,13 +266,13 @@ def gen_data(data):
 
 	wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')}
 
-	for d in data:
+	for d in data['cc']:
 		if d['id'] == 'btc-bitcoin':
 			btcusd = Decimal(d['price_usd'])
 			break
 
 	for k in ('id','symbol'):
-		for d in data:
+		for d in data['cc']:
 			if wants[k]:
 				if d[k] in wants[k]:
 					if d[k] in found[k]:
@@ -149,87 +312,6 @@ def gen_data(data):
 		'last_updated': None,
 	})
 
-def get_src_data(curl_cmd):
-
-	tor_captcha_msg = f"""
-		If you’re using Tor, the API request may have failed due to Captcha protection.
-		A workaround for this issue is to retrieve the JSON data with a browser from
-		the following URL:
-
-		    {api_url}
-
-		and save it to:
-
-		    ‘{cfg.cachedir}/ticker.json’
-
-		Then invoke the program with --cached-data and without --btc
-	"""
-
-	def rate_limit_errmsg(timeout,elapsed):
-		return (
-			f'Rate limit exceeded!  Retry in {timeout-elapsed} seconds' +
-			('' if cfg.btc_only else ', or use --cached-data or --btc')
-		)
-
-	if not os.path.exists(cfg.cachedir):
-		os.makedirs(cfg.cachedir)
-
-	if cfg.btc_only:
-		fn = os.path.join(cfg.cachedir,'ticker-btc.json')
-		timeout = 5 if gcfg.test_suite else btc_ratelimit
-	else:
-		fn = os.path.join(cfg.cachedir,'ticker.json')
-		timeout = 5 if gcfg.test_suite else ratelimit
-
-	fn_rel = os.path.relpath(fn,start=homedir)
-
-	if not os.path.exists(fn):
-		open(fn,'w').write('{}')
-
-	if gcfg.cached_data:
-		json_text = open(fn).read()
-	else:
-		elapsed = int(time.time() - os.stat(fn).st_mtime)
-		if elapsed >= timeout:
-			msg_r(f'Fetching data from {api_host}...')
-			gcfg._util.vmsg('')
-			try:
-				cp = run(curl_cmd,check=True,stdout=PIPE)
-			except CalledProcessError as e:
-				msg('')
-				from .Misc import curl_exit_codes
-				msg(red(curl_exit_codes[e.returncode]))
-				msg(red('Command line:\n  {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) )))
-				from mmgen.exception import MMGenCalledProcessError
-				raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}')
-			json_text = cp.stdout.decode()
-			msg('done')
-		else:
-			die(1,rate_limit_errmsg(timeout,elapsed))
-
-	try:
-		data = json.loads(json_text)
-	except:
-		msg(json_text[:1024] + '...')
-		msg(orange(fmt(tor_captcha_msg,strip_char='\t')))
-		die(2,'Retrieved data is not valid JSON, exiting')
-
-	if not data:
-		if gcfg.cached_data:
-			die(1,'No cached data!  Run command without --cached-data option to retrieve data from remote host')
-		else:
-			die(2,'Remote host returned no data!')
-	elif 'error' in data:
-		die(1,data['error'])
-
-	if gcfg.cached_data:
-		msg(f'Using cached data from ~/{fn_rel}')
-	else:
-		open(fn,'w').write(json_text)
-		msg(f'JSON data cached to ~/{fn_rel}')
-
-	return data
-
 def main():
 
 	def update_sample_file(usr_cfg_file):
@@ -243,20 +325,6 @@ def main():
 				sample_file ))
 			open(sample_file,'w').write(usr_data)
 
-	def get_curl_cmd():
-		return ([
-					'curl',
-					'--tr-encoding',
-					'--compressed', # adds 'Accept-Encoding: gzip'
-					'--header', 'Accept: application/json',
-				] +
-				(['--proxy', cfg.proxy] if cfg.proxy else []) +
-				(['--silent'] if not gcfg.verbose else []) +
-				[api_url + ('/btc-bitcoin' if cfg.btc_only else '')]
-			)
-
-	global cfg,cfg_in
-
 	try:
 		from importlib.resources import files # Python 3.9
 	except ImportError:
@@ -269,17 +337,24 @@ def main():
 		die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format(
 			os.path.relpath(cfg_in.portfolio_file,start=homedir)))
 
-	curl_cmd = get_curl_cmd()
+	if gcfg.list_ids:
+		src_ids = ['cc']
+	elif gcfg.download:
+		assert gcfg.download in DataSource.sources, f'{gcfg.download!r}: invalid data source'
+		src_ids = [gcfg.download]
+	else:
+		src_ids = DataSource.sources
 
-	if gcfg.print_curl:
-		Msg(curl_cmd + '\n' + ' '.join(curl_cmd))
-		return
+	ids = random.sample( list(src_ids), k=len(src_ids) ) # shuffle the ids
 
-	src_data = [get_src_data(curl_cmd)] if cfg.btc_only else get_src_data(curl_cmd)
+	src_data = { k: src_cls[k]().get_data() for k in ids }
+
+	if gcfg.testing:
+		return
 
 	if gcfg.list_ids:
 		from mmgen.ui import do_pager
-		do_pager('\n'.join(e['id'] for e in src_data))
+		do_pager('\n'.join(e['id'] for e in src_data['cc']))
 		return
 
 	global now
@@ -304,10 +379,7 @@ def make_cfg():
 		return tuple(gen())
 
 	def parse_asset_id(s,require_label=False):
-		sym,label = (*s.split('-',1),None)[:2]
-		if require_label and not label:
-			die(1,f'{s!r}: asset label is missing')
-		return asset_tuple( sym.upper(), (s.lower() if label else None) )
+		return src_cls['cc'].parse_asset_id(s,require_label)
 
 	def parse_usr_asset_arg(key,use_cf_file=False):
 		"""
@@ -327,7 +399,8 @@ def make_cfg():
 					None if rate is None else
 					1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else
 					Decimal(rate) ),
-				rate_asset = parse_asset_id(rate_asset) if rate_asset else None )
+				rate_asset = parse_asset_id(rate_asset) if rate_asset else None,
+				source  = parsed_id.source )
 
 		cl_opt = getattr(gcfg,key)
 		cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else []
@@ -344,7 +417,8 @@ def make_cfg():
 				id     = parsed_id.id,
 				amount = None if amount is None else Decimal(amount),
 				rate   = None,
-				rate_asset = None )
+				rate_asset = None,
+				source = parsed_id.source )
 
 		ss = s.split(':')
 		assert len(ss) in (2,3,4), f'{s}: malformed argument'
@@ -417,14 +491,15 @@ def make_cfg():
 		'proxy',
 		'portfolio' ])
 
-	global cfg_in,cfg
+	global cfg_in,src_cls,cfg
+
+	src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() }
 
 	cmd_args = gcfg._args
 	cfg_in = get_cfg_in()
 
 	query_tuple   = namedtuple('query',['asset','to_asset'])
-	asset_data    = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset'])
-	asset_tuple   = namedtuple('asset_tuple',['symbol','id'])
+	asset_data    = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source'])
 
 	usr_rows    = parse_usr_asset_arg('add_rows')
 	usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True)

mmgen_node_tools/main_ticker.py

@@ -12,8 +12,6 @@
 mmnode-ticker: Display price information for cryptocurrency and other assets
 """
 
-from .Ticker import *
-
 opts_data = {
 	'sets': [
 		('wide', True, 'percent_change',  True),
@@ -24,7 +22,7 @@ opts_data = {
 	'text': {
 		'desc':  'Display prices for cryptocurrency and other assets',
 		'usage': '[opts] [TRADE_SPECIFIER]',
-		'options': f"""
+		'options': """
 -h, --help            Print this help message
 --, --longhelp        Print help message for long options (common options)
 -A, --adjust=P        Adjust prices by percentage ‘P’.  In ‘trading’ mode,
@@ -37,6 +35,8 @@ opts_data = {
                       live data from server
 -D, --cachedir=D      Read and write cached JSON data to directory ‘D’
                       instead of ‘~/{dfl_cachedir}’
+-d, --download=D      Retrieve data ‘D’ from source, save to file and exit
+                      (valid options: {ds})
 -e, --add-precision=N Add ‘N’ digits of precision to columns
 -E, --elapsed         Show elapsed time in UPDATED column (see --update-time)
 -F, --portfolio       Display portfolio data
@@ -112,7 +112,7 @@ A TRADE_SPECIFIER is a single argument in the format:
 
                                  PROXY NOTE
 
-The remote server used to obtain the price data, {api_host!r}, blocks
+The remote server used to obtain the price data, {cc.api_host!r}, blocks
 Tor behind a Captcha wall, so a Tor proxy cannot be used directly.  If you’re
 concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then
 set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’
@@ -121,7 +121,7 @@ the script directly on the VPN’ed host with ’proxy’ or --proxy set to the
 null string.
 
 Alternatively, you may download the JSON source data in a Tor-proxied browser
-from ‘{api_url}’, save it as ‘ticker.json’ in your
+from ‘{cc.api_url}’, save it as ‘ticker.json’ in your
 configured cache directory, and run the script with the --cached-data option.
 
 
@@ -130,9 +130,9 @@ configured cache directory, and run the script with the --cached-data option.
 To protect user privacy, all filtering and processing of data is performed
 client side so that the remote server does not know which assets are being
 examined.  This means that data for ALL available assets (currently over 4000)
-is fetched with each invocation of the script.  A rate limit of {L} seconds
+is fetched with each invocation of the script.  A rate limit of {cc.ratelimit} seconds
 between calls is thus imposed to prevent abuse of the remote server.  When the
---btc option is in effect, this limit is reduced to {B} seconds.  To bypass the
+--btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds.  To bypass the
 rate limit entirely, use --cached-data.
 
 
@@ -186,21 +186,20 @@ To add a portfolio, edit the file
 	'code': {
 		'options': lambda s: s.format(
 			dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir),
+			ds           = fmt_dict(DataSource.sources,fmt='equal'),
 		),
 		'notes': lambda s: s.format(
-			assets    = fmt_list(assets_list_gen(cfg_in),fmt='col',indent='  '),
-			cfg       = os.path.relpath(cfg_in.cfg_file,start=homedir),
-			pf_cfg    = os.path.relpath(cfg_in.portfolio_file,start=homedir),
-			api_host  = api_host,
-			api_url   = api_url,
-			L         = ratelimit,
-			B         = btc_ratelimit,
+			assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent='  '),
+			cfg    = os.path.relpath(cfg_in.cfg_file,start=homedir),
+			pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir),
+			cc     = src_cls['cc'](),
 		)
 	}
 }
 
 import os
 
+from mmgen.util import fmt_list,fmt_dict
 from mmgen.cfg import Config
 import mmgen_node_tools.Ticker as tck
 
@@ -208,7 +207,7 @@ tck.gcfg = Config( opts_data=opts_data, do_post_init=True )
 
 tck.make_cfg()
 
-from .Ticker import cfg_in
+from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls
 
 tck.gcfg._post_init()