Ticker.py 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100
  1. #!/usr/bin/env python3
  2. #
  3. # mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet
  4. # Copyright (C)2013-2022 The MMGen Project <mmgen@tuta.io>
  5. # Licensed under the GNU General Public License, Version 3:
  6. # https://www.gnu.org/licenses
  7. # Public project repositories:
  8. # https://github.com/mmgen/mmgen-wallet https://github.com/mmgen/mmgen-node-tools
  9. # https://gitlab.com/mmgen/mmgen-wallet https://gitlab.com/mmgen/mmgen-node-tools
  10. """
  11. mmgen_node_tools.Ticker: Display price information for cryptocurrency and other assets
  12. """
  13. # v3.2.dev4: switch to new coinpaprika ‘tickers’ API call (supports ‘limit’ parameter, more historical data)
  14. # Old ‘ticker’ API (/v1/ticker): data['BTC']['price_usd']
  15. # New ‘tickers’ API (/v1/tickers): data['BTC']['quotes']['USD']['price']
  16. # Possible alternatives:
  17. # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR
  18. import sys,os,re,time,datetime,json,yaml,random
  19. from subprocess import run,PIPE,CalledProcessError
  20. from decimal import Decimal
  21. from collections import namedtuple
  22. from mmgen.color import red,yellow,green,blue,orange,gray
  23. from mmgen.util import msg,msg_r,Msg,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen
  24. homedir = os.getenv('HOME')
  25. dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools')
  26. cfg_fn = 'ticker-cfg.yaml'
  27. portfolio_fn = 'ticker-portfolio.yaml'
  28. asset_tuple = namedtuple('asset_tuple',['symbol','id','source'])
  29. last_api_host = None
  30. percent_cols = {
  31. 'd': 'day',
  32. 'w': 'week',
  33. 'm': 'month',
  34. 'y': 'year',
  35. }
  36. class DataSource:
  37. source_groups = [
  38. {
  39. 'cc': 'coinpaprika'
  40. }, {
  41. 'fi': 'yahoospot',
  42. 'hi': 'yahoohist',
  43. }
  44. ]
  45. @classmethod
  46. def get_sources(cls,randomize=False):
  47. g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups
  48. return {k:v for a in g for k,v in a.items()}
  49. class base:
  50. def fetch_delay(self):
  51. global last_api_host
  52. if not gcfg.testing and last_api_host and last_api_host != self.api_host:
  53. delay = 1 + random.randrange(1,5000) / 1000
  54. msg_r(f'Waiting {delay:.3f} seconds...')
  55. time.sleep(delay)
  56. msg('')
  57. last_api_host = self.api_host
  58. def get_data_from_network(self):
  59. curl_cmd = list_gen(
  60. ['curl', '--tr-encoding', '--header', 'Accept: application/json',True],
  61. ['--compressed'], # adds 'Accept-Encoding: gzip'
  62. ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)],
  63. ['--silent', not gcfg.verbose],
  64. [self.api_url]
  65. )
  66. if gcfg.testing:
  67. Msg(fmt_list(curl_cmd,fmt='bare'))
  68. return
  69. try:
  70. return run(curl_cmd,check=True,stdout=PIPE).stdout.decode()
  71. except CalledProcessError as e:
  72. msg('')
  73. from .Misc import curl_exit_codes
  74. msg(red(curl_exit_codes[e.returncode]))
  75. msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) )))
  76. from mmgen.exception import MMGenCalledProcessError
  77. raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}')
  78. def get_data(self):
  79. if not os.path.exists(cfg.cachedir):
  80. os.makedirs(cfg.cachedir)
  81. if not os.path.exists(self.json_fn):
  82. open(self.json_fn,'w').write('{}')
  83. if gcfg.cached_data:
  84. data_type = 'json'
  85. data_in = open(self.json_fn).read()
  86. else:
  87. data_type = self.net_data_type
  88. elapsed = int(time.time() - os.stat(self.json_fn).st_mtime)
  89. if elapsed >= self.timeout or gcfg.testing:
  90. if gcfg.testing:
  91. msg('')
  92. self.fetch_delay()
  93. msg_r(f'Fetching {self.data_desc} from {self.api_host}...')
  94. if self.has_verbose:
  95. gcfg._util.vmsg('')
  96. data_in = self.get_data_from_network()
  97. msg('done')
  98. if gcfg.testing:
  99. return {}
  100. else:
  101. die(1,self.rate_limit_errmsg(elapsed))
  102. if data_type == 'json':
  103. try:
  104. data = json.loads(data_in)
  105. except:
  106. self.json_data_error_msg(data_in)
  107. die(2,'Retrieved data is not valid JSON, exiting')
  108. json_text = data_in
  109. elif data_type == 'python':
  110. data = data_in
  111. json_text = json.dumps(data_in)
  112. if not data:
  113. if gcfg.cached_data:
  114. die(1,'No cached data! Run command without --cached-data option to retrieve data from remote host')
  115. else:
  116. die(2,'Remote host returned no data!')
  117. elif 'error' in data:
  118. die(1,data['error'])
  119. if gcfg.cached_data:
  120. msg(f'Using cached data from ~/{self.json_fn_rel}')
  121. else:
  122. open(self.json_fn,'w').write(json_text)
  123. msg(f'JSON data cached to ~/{self.json_fn_rel}')
  124. if gcfg.download:
  125. sys.exit(0)
  126. return self.postprocess_data(data)
  127. def json_data_error_msg(self,json_text):
  128. pass
  129. def postprocess_data(self,data):
  130. return data
  131. @property
  132. def json_fn_rel(self):
  133. return os.path.relpath(self.json_fn,start=homedir)
  134. class coinpaprika(base):
  135. desc = 'CoinPaprika'
  136. data_desc = 'cryptocurrency data'
  137. api_host = 'api.coinpaprika.com'
  138. ratelimit = 240
  139. btc_ratelimit = 10
  140. net_data_type = 'json'
  141. has_verbose = True
  142. dfl_asset_limit = 2000
  143. def __init__(self):
  144. self.asset_limit = int(gcfg.asset_limit or self.dfl_asset_limit)
  145. def rate_limit_errmsg(self,elapsed):
  146. return (
  147. f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' +
  148. ('' if cfg.btc_only else ', or use --cached-data or --btc')
  149. )
  150. @property
  151. def api_url(self):
  152. return (
  153. f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else
  154. f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else
  155. f'https://{self.api_host}/v1/tickers' )
  156. @property
  157. def json_fn(self):
  158. return os.path.join(
  159. cfg.cachedir,
  160. 'ticker-btc.json' if cfg.btc_only else 'ticker.json' )
  161. @property
  162. def timeout(self):
  163. return 5 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit
  164. def json_data_error_msg(self,json_text):
  165. tor_captcha_msg = f"""
  166. If you’re using Tor, the API request may have failed due to Captcha protection.
  167. A workaround for this issue is to retrieve the JSON data with a browser from
  168. the following URL:
  169. {self.api_url}
  170. and save it to:
  171. ‘{cfg.cachedir}/ticker.json’
  172. Then invoke the program with --cached-data and without --btc
  173. """
  174. msg(json_text[:1024] + '...')
  175. msg(orange(fmt(tor_captcha_msg,strip_char='\t')))
  176. def postprocess_data(self,data):
  177. return [data] if cfg.btc_only else data
  178. @staticmethod
  179. def parse_asset_id(s,require_label):
  180. sym,label = (*s.split('-',1),None)[:2]
  181. if require_label and not label:
  182. die(1,f'{s!r}: asset label is missing')
  183. return asset_tuple(
  184. symbol = sym.upper(),
  185. id = (s.lower() if label else None),
  186. source = 'cc' )
  187. class yahoospot(base):
  188. desc = 'Yahoo Finance'
  189. data_desc = 'spot financial data'
  190. api_host = 'finance.yahoo.com'
  191. ratelimit = 30
  192. net_data_type = 'python'
  193. has_verbose = False
  194. asset_id_pat = r'^\^.*|.*=[xf]$'
  195. json_fn_basename = 'ticker-finance.json'
  196. @staticmethod
  197. def get_id(sym,data):
  198. return sym.lower()
  199. @staticmethod
  200. def conv_data(sym,data,btcusd):
  201. price_usd = Decimal( data['regularMarketPrice']['raw'] )
  202. return {
  203. 'id': sym,
  204. 'name': data['shortName'],
  205. 'symbol': sym.upper(),
  206. 'price_usd': price_usd,
  207. 'price_btc': price_usd / btcusd,
  208. 'percent_change_1y': data['pct_chg_1y'],
  209. 'percent_change_30d': data['pct_chg_4wks'],
  210. 'percent_change_7d': data['pct_chg_1wk'],
  211. 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100,
  212. 'last_updated': data['regularMarketTime'],
  213. }
  214. def rate_limit_errmsg(self,elapsed):
  215. return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data'
  216. @property
  217. def json_fn(self):
  218. return os.path.join( cfg.cachedir, self.json_fn_basename )
  219. @property
  220. def timeout(self):
  221. return 5 if gcfg.test_suite else self.ratelimit
  222. @property
  223. def symbols(self):
  224. return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi']
  225. def get_data_from_network(self):
  226. kwargs = {
  227. 'formatted': True,
  228. 'asynchronous': True,
  229. 'proxies': { 'https': cfg.proxy2 },
  230. }
  231. if gcfg.test_suite:
  232. kwargs.update({ 'timeout': 1, 'retry': 0 })
  233. if gcfg.testing:
  234. Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format(
  235. self.symbols,
  236. fmt_dict(kwargs,fmt='kwargs') ))
  237. return
  238. from yahooquery import Ticker
  239. return self.process_network_data( Ticker(self.symbols,**kwargs) )
  240. def process_network_data(self,ticker):
  241. return ticker.price
  242. @staticmethod
  243. def parse_asset_id(s,require_label):
  244. return asset_tuple(
  245. symbol = s.upper(),
  246. id = s.lower(),
  247. source = 'fi' )
  248. class yahoohist(yahoospot):
  249. json_fn_basename = 'ticker-finance-history.json'
  250. data_desc = 'historical financial data'
  251. net_data_type = 'json'
  252. period = '1y'
  253. interval = '1wk'
  254. def process_network_data(self,ticker):
  255. return ticker.history(
  256. period = self.period,
  257. interval = self.interval).to_json(orient='index')
  258. def postprocess_data(self,data):
  259. def gen():
  260. keys = set()
  261. for key,val in data.items():
  262. if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key):
  263. date = '{}-{:>02}-{:>02}'.format(*m[2].split(', '))
  264. if (sym := m[1]) in keys:
  265. d[date] = val
  266. else:
  267. keys.add(sym)
  268. d = {date:val}
  269. yield (sym,d)
  270. return dict(gen())
  271. def assets_list_gen(cfg_in):
  272. for k,v in cfg_in.cfg['assets'].items():
  273. yield ''
  274. yield k.upper()
  275. for e in v:
  276. out = e.split('-',1)
  277. yield ' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '')
  278. def gen_data(data):
  279. """
  280. Filter the raw data and return it as a dict keyed by the IDs of the assets
  281. we want to display.
  282. Add dummy entry for USD and entry for user-specified asset, if any.
  283. Since symbols in source data are not guaranteed to be unique (e.g. XAG), we
  284. must search the data twice: first for unique IDs, then for symbols while
  285. checking for duplicates.
  286. """
  287. def dup_sym_errmsg(dup_sym):
  288. return (
  289. f'The symbol {dup_sym!r} is shared by the following assets:\n' +
  290. '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) +
  291. '\n\nPlease specify the asset by one of the full IDs listed above\n' +
  292. f'instead of {dup_sym!r}'
  293. )
  294. def check_assets_found(wants,found,keys=['symbol','id']):
  295. error = False
  296. for k in keys:
  297. missing = wants[k] - found[k]
  298. if missing:
  299. msg(
  300. ('The following IDs were not found in source data:\n{}' if k == 'id' else
  301. 'The following symbols could not be resolved:\n{}').format(
  302. fmt_list(missing,fmt='col',indent=' ')
  303. ))
  304. error = True
  305. if error:
  306. die(1,'Missing data, exiting')
  307. rows_want = {
  308. 'id': {r.id for r in cfg.rows if isinstance(r,tuple) and r.id} - {'usd-us-dollar'},
  309. 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'},
  310. }
  311. usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset)
  312. usr_rate_assets_want = {
  313. 'id': {a.id for a in usr_rate_assets if a.id},
  314. 'symbol': {a.symbol for a in usr_rate_assets if not a.id}
  315. }
  316. usr_assets = cfg.usr_rows + cfg.usr_columns + tuple(c for c in (cfg.query or ()) if c)
  317. usr_wants = {
  318. 'id': (
  319. {a.id for a in usr_assets + usr_rate_assets if a.id} -
  320. {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'} )
  321. ,
  322. 'symbol': (
  323. {a.symbol for a in usr_assets + usr_rate_assets if not a.id} -
  324. {a.symbol for a in usr_assets if a.rate} - {'USD'} ),
  325. }
  326. found = { 'id': set(), 'symbol': set() }
  327. rate_assets = {}
  328. wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')}
  329. for d in data['cc']:
  330. if d['id'] == 'btc-bitcoin':
  331. btcusd = Decimal(str(d['quotes']['USD']['price']))
  332. break
  333. get_id = src_cls['fi'].get_id
  334. conv_func = src_cls['fi'].conv_data
  335. for k,v in data['fi'].items():
  336. id = get_id(k,v)
  337. if wants['id']:
  338. if id in wants['id']:
  339. if id in found['id']:
  340. die(1,dup_sym_errmsg(id))
  341. if m := data['hi'].get(k):
  342. spot = v['regularMarketPrice']['raw']
  343. hist = tuple(m.values())
  344. v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = (
  345. (spot / hist[-2]['close'] - 1) * 100,
  346. (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month
  347. (spot / hist[0]['close'] - 1) * 100,
  348. )
  349. else:
  350. v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None
  351. yield ( id, conv_func(id,v,btcusd) )
  352. found['id'].add(id)
  353. wants['id'].remove(id)
  354. if id in usr_rate_assets_want['id']:
  355. rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key
  356. else:
  357. break
  358. for k in ('id','symbol'):
  359. for d in data['cc']:
  360. if wants[k]:
  361. if d[k] in wants[k]:
  362. if d[k] in found[k]:
  363. die(1,dup_sym_errmsg(d[k]))
  364. if not 'price_usd' in d:
  365. d['price_usd'] = Decimal(str(d['quotes']['USD']['price']))
  366. d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd
  367. d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h']
  368. d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d']
  369. d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d']
  370. d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y']
  371. d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated']).timestamp())
  372. yield (d['id'],d)
  373. found[k].add(d[k])
  374. wants[k].remove(d[k])
  375. if d[k] in usr_rate_assets_want[k]:
  376. rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key
  377. else:
  378. break
  379. check_assets_found(usr_wants,found)
  380. for asset in (cfg.usr_rows + cfg.usr_columns):
  381. if asset.rate:
  382. """
  383. User-supplied rate overrides rate from source data.
  384. """
  385. _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower()
  386. ra_rate = rate_assets[asset.rate_asset.symbol]['price_usd'] if asset.rate_asset else 1
  387. yield ( _id, {
  388. 'symbol': asset.symbol,
  389. 'id': _id,
  390. 'name': ' '.join(_id.split('-')[1:]),
  391. 'price_usd': ra_rate / asset.rate,
  392. 'price_btc': ra_rate / asset.rate / btcusd,
  393. 'last_updated': None,
  394. })
  395. yield ('usd-us-dollar', {
  396. 'symbol': 'USD',
  397. 'id': 'usd-us-dollar',
  398. 'name': 'US Dollar',
  399. 'price_usd': Decimal(1),
  400. 'price_btc': Decimal(1) / btcusd,
  401. 'last_updated': None,
  402. })
  403. def main():
  404. def update_sample_file(usr_cfg_file):
  405. usr_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text()
  406. sample_file = usr_cfg_file + '.sample'
  407. sample_data = open(sample_file).read() if os.path.exists(sample_file) else None
  408. if usr_data != sample_data:
  409. os.makedirs(os.path.dirname(sample_file),exist_ok=True)
  410. msg('{} {}'.format(
  411. ('Updating','Creating')[sample_data is None],
  412. sample_file ))
  413. open(sample_file,'w').write(usr_data)
  414. try:
  415. from importlib.resources import files # Python 3.9
  416. except ImportError:
  417. from importlib_resources import files
  418. update_sample_file(cfg_in.cfg_file)
  419. update_sample_file(cfg_in.portfolio_file)
  420. if gcfg.portfolio and not cfg_in.portfolio:
  421. die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format(
  422. os.path.relpath(cfg_in.portfolio_file,start=homedir)))
  423. if gcfg.list_ids:
  424. src_ids = ['cc']
  425. elif gcfg.download:
  426. if not gcfg.download in DataSource.get_sources():
  427. die(1,f'{gcfg.download!r}: invalid data source')
  428. src_ids = [gcfg.download]
  429. else:
  430. src_ids = DataSource.get_sources(randomize=True)
  431. src_data = { k: src_cls[k]().get_data() for k in src_ids }
  432. if gcfg.testing:
  433. return
  434. if gcfg.list_ids:
  435. from mmgen.ui import do_pager
  436. do_pager('\n'.join(e['id'] for e in src_data['cc']))
  437. return
  438. global now
  439. now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900
  440. data = dict(gen_data(src_data))
  441. gcfg._util.stdout_or_pager(
  442. '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n'
  443. )
  444. def make_cfg(gcfg_arg):
  445. query_tuple = namedtuple('query',['asset','to_asset'])
  446. asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source'])
  447. def parse_asset_id(s,require_label=False):
  448. return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label)
  449. def get_rows_from_cfg(add_data=None):
  450. def gen():
  451. for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()):
  452. yield k
  453. if add_data and k in add_data:
  454. v += tuple(add_data[k])
  455. for e in v:
  456. yield parse_asset_id(e,require_label=True)
  457. return tuple(gen())
  458. def parse_percent_cols(arg):
  459. if arg is None:
  460. return []
  461. res = arg.lower().split(',')
  462. for s in res:
  463. if s not in percent_cols:
  464. die(1,f'{arg!r}: invalid --percent-cols parameter (valid letters: {fmt_list(percent_cols)})')
  465. return res
  466. def parse_usr_asset_arg(key,use_cf_file=False):
  467. """
  468. asset_id[:rate[:rate_asset]]
  469. """
  470. def parse_parm(s):
  471. ss = s.split(':')
  472. assert len(ss) in (1,2,3), f'{s}: malformed argument'
  473. asset_id,rate,rate_asset = (*ss,None,None)[:3]
  474. parsed_id = parse_asset_id(asset_id)
  475. return asset_data(
  476. symbol = parsed_id.symbol,
  477. id = parsed_id.id,
  478. amount = None,
  479. rate = (
  480. None if rate is None else
  481. 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else
  482. Decimal(rate) ),
  483. rate_asset = parse_asset_id(rate_asset) if rate_asset else None,
  484. source = parsed_id.source )
  485. cl_opt = getattr(gcfg,key)
  486. cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else []
  487. return tuple( parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt) )
  488. def parse_query_arg(s):
  489. """
  490. asset_id:amount[:to_asset_id[:to_amount]]
  491. """
  492. def parse_query_asset(asset_id,amount):
  493. parsed_id = parse_asset_id(asset_id)
  494. return asset_data(
  495. symbol = parsed_id.symbol,
  496. id = parsed_id.id,
  497. amount = None if amount is None else Decimal(amount),
  498. rate = None,
  499. rate_asset = None,
  500. source = parsed_id.source )
  501. ss = s.split(':')
  502. assert len(ss) in (2,3,4), f'{s}: malformed argument'
  503. asset_id,amount,to_asset_id,to_amount = (*ss,None,None)[:4]
  504. return query_tuple(
  505. asset = parse_query_asset(asset_id,amount),
  506. to_asset = parse_query_asset(to_asset_id,to_amount) if to_asset_id else None
  507. )
  508. def gen_uniq(obj_list,key,preload=None):
  509. found = set([getattr(obj,key) for obj in preload if hasattr(obj,key)] if preload else ())
  510. for obj in obj_list:
  511. id = getattr(obj,key)
  512. if id not in found:
  513. yield obj
  514. found.add(id)
  515. def get_usr_assets():
  516. return (
  517. 'user_added',
  518. usr_rows +
  519. (tuple(asset for asset in query if asset) if query else ()) +
  520. usr_columns )
  521. def get_portfolio_assets(ret=()):
  522. if cfg_in.portfolio and gcfg.portfolio:
  523. ret = (parse_asset_id(e,require_label=True) for e in cfg_in.portfolio)
  524. return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') )
  525. def get_portfolio():
  526. return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'}
  527. def parse_add_precision(s):
  528. if not s:
  529. return 0
  530. if not (s.isdigit() and s.isascii()):
  531. die(1,f'{s}: invalid parameter for --add-precision (not an integer)')
  532. if int(s) > 30:
  533. die(1,f'{s}: invalid parameter for --add-precision (value >30)')
  534. return int(s)
  535. def create_rows():
  536. rows = (
  537. ('trade_pair',) + query if (query and query.to_asset) else
  538. ('bitcoin',parse_asset_id('btc-bitcoin')) if gcfg.btc else
  539. get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None )
  540. )
  541. for hdr,data in (
  542. (get_usr_assets(),) if query else
  543. (get_usr_assets(), get_portfolio_assets())
  544. ):
  545. if data:
  546. uniq_data = tuple(gen_uniq(data,'symbol',preload=rows))
  547. if uniq_data:
  548. rows += (hdr,) + uniq_data
  549. return rows
  550. cfg_tuple = namedtuple('global_cfg',[
  551. 'rows',
  552. 'usr_rows',
  553. 'usr_columns',
  554. 'query',
  555. 'adjust',
  556. 'clsname',
  557. 'btc_only',
  558. 'add_prec',
  559. 'cachedir',
  560. 'proxy',
  561. 'proxy2',
  562. 'portfolio',
  563. 'percent_cols' ])
  564. global gcfg,cfg_in,src_cls,cfg
  565. gcfg = gcfg_arg
  566. src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() }
  567. fi_pat = src_cls['fi'].asset_id_pat
  568. cmd_args = gcfg._args
  569. cfg_in = get_cfg_in()
  570. usr_rows = parse_usr_asset_arg('add_rows')
  571. usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True)
  572. query = parse_query_arg(cmd_args[0]) if cmd_args else None
  573. def get_proxy(name):
  574. proxy = getattr(gcfg,name)
  575. return (
  576. '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none')
  577. else (proxy or cfg_in.cfg.get(name))
  578. )
  579. proxy = get_proxy('proxy')
  580. proxy = None if proxy == 'none' else proxy
  581. proxy2 = get_proxy('proxy2')
  582. cfg = cfg_tuple(
  583. rows = create_rows(),
  584. usr_rows = usr_rows,
  585. usr_columns = usr_columns,
  586. query = query,
  587. adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ),
  588. clsname = 'trading' if query else 'overview',
  589. btc_only = gcfg.btc,
  590. add_prec = parse_add_precision(gcfg.add_precision),
  591. cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir,
  592. proxy = proxy,
  593. proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy),
  594. portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None,
  595. percent_cols = parse_percent_cols(gcfg.percent_cols)
  596. )
  597. def get_cfg_in():
  598. ret = namedtuple('cfg_in_data',['cfg','portfolio','cfg_file','portfolio_file'])
  599. cfg_file,portfolio_file = (
  600. [os.path.join(gcfg.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)]
  601. )
  602. cfg_data,portfolio_data = (
  603. [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None for fn in (cfg_file,portfolio_file)]
  604. )
  605. return ret(
  606. cfg = cfg_data or {
  607. 'assets': {
  608. 'coin': [ 'btc-bitcoin', 'eth-ethereum', 'xmr-monero' ],
  609. # gold futures, silver futures, Brent futures
  610. 'commodity': [ 'gc=f', 'si=f', 'bz=f' ],
  611. # Pound Sterling, Euro, Swiss Franc
  612. 'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ],
  613. # Dow Jones Industrials, Nasdaq 100, S&P 500
  614. 'index': [ '^dji', '^ixic', '^gspc' ],
  615. },
  616. 'proxy': 'http://vpn-gw:8118'
  617. },
  618. portfolio = portfolio_data,
  619. cfg_file = cfg_file,
  620. portfolio_file = portfolio_file,
  621. )
  622. class Ticker:
  623. class base:
  624. offer = None
  625. to_asset = None
  626. def __init__(self,data):
  627. self.comma = ',' if gcfg.thousands_comma else ''
  628. self.col1_wid = max(len('TOTAL'),(
  629. max(len(self.create_label(d['id'])) for d in data.values()) if gcfg.name_labels else
  630. max(len(d['symbol']) for d in data.values())
  631. )) + 1
  632. self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows]
  633. self.col_usd_prices = {k:self.data[k]['price_usd'] for k in self.col_ids}
  634. self.prices = {row.id:self.get_row_prices(row.id)
  635. for row in self.rows if isinstance(row,tuple) and row.id in data}
  636. self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar')
  637. def format_last_update_col(self,cross_assets=()):
  638. if gcfg.elapsed:
  639. from mmgen.util2 import format_elapsed_hr
  640. fmt_func = format_elapsed_hr
  641. else:
  642. fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t))
  643. d = self.data
  644. max_w = 0
  645. if cross_assets:
  646. last_updated_x = [d[a.id]['last_updated'] for a in cross_assets]
  647. min_t = min( (int(n) for n in last_updated_x if isinstance(n,int) ), default=None )
  648. else:
  649. min_t = None
  650. for row in self.rows:
  651. if isinstance(row,tuple):
  652. try:
  653. t = int( d[row.id]['last_updated'] )
  654. except TypeError as e:
  655. d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e))
  656. except KeyError as e:
  657. msg(str(e))
  658. pass
  659. else:
  660. t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now )
  661. max_w = max(len(t_fmt),max_w)
  662. self.upd_w = max_w
  663. def init_prec(self):
  664. exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted() ) for a in self.usr_col_assets]
  665. self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp }
  666. self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp }
  667. def get_id(self,asset):
  668. if asset.id:
  669. return asset.id
  670. else:
  671. for d in self.data.values():
  672. if d['symbol'] == asset.symbol:
  673. return d['id']
  674. def create_label(self,id):
  675. return self.data[id]['name'].upper()
  676. def gen_output(self):
  677. yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now)))
  678. for asset in self.usr_col_assets:
  679. if asset.symbol != 'USD':
  680. usdprice = self.data[asset.id]['price_usd']
  681. yield '{} ({}) = {:{}.{}f} USD'.format(
  682. asset.symbol,
  683. self.create_label(asset.id),
  684. usdprice,
  685. self.comma,
  686. max(2, 4-usdprice.adjusted()) )
  687. if hasattr(self,'subhdr'):
  688. yield self.subhdr
  689. if self.show_adj:
  690. yield (
  691. ('Offered price differs from spot' if self.offer else 'Adjusting prices')
  692. + ' by '
  693. + yellow('{:+.2f}%'.format( (self.adjust-1) * 100 ))
  694. )
  695. yield ''
  696. if cfg.portfolio:
  697. yield blue('PRICES')
  698. if self.table_hdr:
  699. yield self.table_hdr
  700. for row in self.rows:
  701. if isinstance(row,str):
  702. yield ('-' * self.hl_wid)
  703. else:
  704. try:
  705. yield self.fmt_row(self.data[row.id])
  706. except KeyError:
  707. yield gray(f'(no data for {row.id})')
  708. yield '-' * self.hl_wid
  709. if cfg.portfolio:
  710. self.fs_num = self.fs_num2
  711. self.fs_str = self.fs_str2
  712. yield ''
  713. yield blue('PORTFOLIO')
  714. yield self.table_hdr
  715. yield '-' * self.hl_wid
  716. for sym,amt in cfg.portfolio.items():
  717. try:
  718. yield self.fmt_row(self.data[sym],amt=amt)
  719. except KeyError:
  720. yield gray(f'(no data for {sym})')
  721. yield '-' * self.hl_wid
  722. if not cfg.btc_only:
  723. yield self.fs_num.format(
  724. lbl = 'TOTAL', pc3='', pc4='', pc1='', pc2='', upd='', amt='',
  725. **{ k.replace('-','_'): v for k,v in self.prices['total'].items() }
  726. )
  727. class overview(base):
  728. def __init__(self,data):
  729. self.data = data
  730. self.adjust = cfg.adjust
  731. self.show_adj = self.adjust != 1
  732. self.usr_col_assets = [asset._replace(id=self.get_id(asset)) for asset in cfg.usr_columns]
  733. self.col_ids = ('usd-us-dollar',) + tuple(a.id for a in self.usr_col_assets) + ('btc-bitcoin',)
  734. super().__init__(data)
  735. self.format_last_update_col()
  736. if cfg.portfolio:
  737. self.prices['total'] = { col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id]
  738. for row in self.rows if isinstance(row,tuple) and row.id in cfg.portfolio and row.id in data)
  739. for col_id in self.col_ids }
  740. self.init_prec()
  741. self.init_fs()
  742. def get_row_prices(self,id):
  743. if id in self.data:
  744. d = self.data[id]
  745. return { k: (
  746. d['price_btc'] if k == 'btc-bitcoin' else
  747. d['price_usd'] / self.col_usd_prices[k]
  748. ) * self.adjust for k in self.col_ids }
  749. def fmt_row(self,d,amt=None,amt_fmt=None):
  750. def fmt_pct(n):
  751. return gray(' --') if n == None else (red,green)[n>=0](f'{n:+7.2f}')
  752. p = self.prices[d['id']]
  753. if amt is not None:
  754. amt_fmt = f'{amt:{19+cfg.add_prec}{self.comma}.{8+cfg.add_prec}f}'
  755. if '.' in amt_fmt:
  756. amt_fmt = amt_fmt.rstrip('0').rstrip('.')
  757. return self.fs_num.format(
  758. lbl = (self.create_label(d['id']) if gcfg.name_labels else d['symbol']),
  759. pc1 = fmt_pct(d.get('percent_change_7d')),
  760. pc2 = fmt_pct(d.get('percent_change_24h')),
  761. pc3 = fmt_pct(d.get('percent_change_1y')),
  762. pc4 = fmt_pct(d.get('percent_change_30d')),
  763. upd = d.get('last_updated_fmt'),
  764. amt = amt_fmt,
  765. **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() }
  766. )
  767. def init_fs(self):
  768. col_prec = {'usd-us-dollar':2+cfg.add_prec,'btc-bitcoin':8+cfg.add_prec } # | self.uprec # Python 3.9
  769. col_prec.update(self.uprec)
  770. col_wid = {'usd-us-dollar':8+cfg.add_prec,'btc-bitcoin':12+cfg.add_prec } # """
  771. col_wid.update(self.uwid)
  772. max_row = max(
  773. ( (k,v['btc-bitcoin']) for k,v in self.prices.items() ),
  774. key = lambda a: a[1]
  775. )
  776. widths = { k: len('{:{}.{}f}'.format( self.prices[max_row[0]][k], self.comma, col_prec[k] ))
  777. for k in self.col_ids }
  778. fd = namedtuple('format_str_data',['fs_str','fs_num','wid'])
  779. col_fs_data = {
  780. 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid),
  781. 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8),
  782. 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8),
  783. 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8),
  784. 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8),
  785. 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2),
  786. 'amt': fd(' {amt}', ' {amt}', 21),
  787. }
  788. # } | { k: fd( # Python 3.9
  789. col_fs_data.update({ k: fd(
  790. ' {{{}:>{}}}'.format( k.replace('-','_'), widths[k] ),
  791. ' {{{}:{}{}.{}f}}'.format( k.replace('-','_'), widths[k], self.comma, col_prec[k] ),
  792. widths[k]+2
  793. ) for k in self.col_ids
  794. })
  795. cols = (
  796. ['label','usd-us-dollar'] +
  797. [asset.id for asset in self.usr_col_assets] +
  798. [a for a,b in (
  799. ( 'btc-bitcoin', not cfg.btc_only ),
  800. ( 'pct1y', 'y' in cfg.percent_cols ),
  801. ( 'pct1m', 'm' in cfg.percent_cols ),
  802. ( 'pct1w', 'w' in cfg.percent_cols ),
  803. ( 'pct1d', 'd' in cfg.percent_cols ),
  804. ( 'update_time', gcfg.update_time ),
  805. ) if b]
  806. )
  807. cols2 = list(cols)
  808. if gcfg.update_time:
  809. cols2.pop()
  810. cols2.append('amt')
  811. self.fs_str = ''.join(col_fs_data[c].fs_str for c in cols)
  812. self.fs_num = ''.join(col_fs_data[c].fs_num for c in cols)
  813. self.hl_wid = sum(col_fs_data[c].wid for c in cols)
  814. self.fs_str2 = ''.join(col_fs_data[c].fs_str for c in cols2)
  815. self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2)
  816. self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2)
  817. @property
  818. def table_hdr(self):
  819. return self.fs_str.format(
  820. lbl = '',
  821. pc1 = ' CHG_7d',
  822. pc2 = 'CHG_24h',
  823. pc3 = 'CHG_1y',
  824. pc4 = 'CHG_30d',
  825. upd = 'UPDATED',
  826. amt = ' AMOUNT',
  827. usd_us_dollar = 'USD',
  828. btc_bitcoin = ' BTC',
  829. **{ a.id.replace('-','_'): a.symbol for a in self.usr_col_assets }
  830. )
  831. class trading(base):
  832. def __init__(self,data):
  833. self.data = data
  834. self.asset = cfg.query.asset._replace(id=self.get_id(cfg.query.asset))
  835. self.to_asset = (
  836. cfg.query.to_asset._replace(id=self.get_id(cfg.query.to_asset))
  837. if cfg.query.to_asset else None )
  838. self.col_ids = [self.asset.id]
  839. self.adjust = cfg.adjust
  840. if self.to_asset:
  841. self.offer = self.to_asset.amount
  842. if self.offer:
  843. real_price = (
  844. self.asset.amount
  845. * data[self.asset.id]['price_usd']
  846. / data[self.to_asset.id]['price_usd']
  847. )
  848. if self.adjust != 1:
  849. die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier')
  850. self.adjust = self.offer / real_price
  851. self.hl_ids = [self.asset.id,self.to_asset.id]
  852. else:
  853. self.hl_ids = [self.asset.id]
  854. self.show_adj = self.adjust != 1 or self.offer
  855. super().__init__(data)
  856. self.usr_col_assets = [self.asset] + ([self.to_asset] if self.to_asset else [])
  857. for a in self.usr_col_assets:
  858. self.prices[a.id]['usd-us-dollar'] = data[a.id]['price_usd']
  859. self.format_last_update_col(cross_assets=self.usr_col_assets)
  860. self.init_prec()
  861. self.init_fs()
  862. def get_row_prices(self,id):
  863. if id in self.data:
  864. d = self.data[id]
  865. return { k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids }
  866. def init_fs(self):
  867. self.max_wid = max(
  868. len('{:{}{}.{}f}'.format(
  869. v[self.asset.id] * self.asset.amount,
  870. 16 + cfg.add_prec,
  871. self.comma,
  872. 8 + cfg.add_prec
  873. ))
  874. for v in self.prices.values()
  875. )
  876. self.fs_str = '{lbl:%s} {p_spot}' % self.col1_wid
  877. self.hl_wid = self.col1_wid + self.max_wid + 1
  878. if self.show_adj:
  879. self.fs_str += ' {p_adj}'
  880. self.hl_wid += self.max_wid + 1
  881. if gcfg.update_time:
  882. self.fs_str += ' {upd}'
  883. self.hl_wid += self.upd_w + 2
  884. def fmt_row(self,d):
  885. id = d['id']
  886. p = self.prices[id][self.asset.id] * self.asset.amount
  887. p_spot = '{:{}{}.{}f}'.format( p, self.max_wid, self.comma, 8+cfg.add_prec )
  888. p_adj = (
  889. '{:{}{}.{}f}'.format( p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec )
  890. if self.show_adj else '' )
  891. return self.fs_str.format(
  892. lbl = (self.create_label(id) if gcfg.name_labels else d['symbol']),
  893. p_spot = green(p_spot) if id in self.hl_ids else p_spot,
  894. p_adj = yellow(p_adj) if id in self.hl_ids else p_adj,
  895. upd = d.get('last_updated_fmt'),
  896. )
  897. @property
  898. def table_hdr(self):
  899. return self.fs_str.format(
  900. lbl = '',
  901. p_spot = '{t:>{w}}'.format(
  902. t = 'SPOT PRICE',
  903. w = self.max_wid ),
  904. p_adj = '{t:>{w}}'.format(
  905. t = ('OFFERED' if self.offer else 'ADJUSTED') + ' PRICE',
  906. w = self.max_wid ),
  907. upd = 'UPDATED'
  908. )
  909. @property
  910. def subhdr(self):
  911. return (
  912. '{a}: {b:{c}} {d}'.format(
  913. a = 'Offer' if self.offer else 'Amount',
  914. b = self.asset.amount,
  915. c = self.comma,
  916. d = self.asset.symbol
  917. ) + (
  918. (
  919. ' =>' +
  920. (' {:{}}'.format(self.offer,self.comma) if self.offer else '') +
  921. ' {} ({})'.format(
  922. self.to_asset.symbol,
  923. self.create_label(self.to_asset.id) )
  924. ) if self.to_asset else '' )
  925. )