mmnode-ticker: retrieve financial data from Yahoo Finance

mmgen_node_tools/Ticker.py

@@ -46,6 +46,7 @@ class DataSource:
 
 	sources = {
 		'cc': 'coinpaprika',
+		'fi': 'yahooquery'
 	}
 
 	class base:
@@ -199,6 +200,69 @@ class DataSource:
 				id     = (s.lower() if label else None),
 				source = 'cc' )
 
+	class yahooquery(base):
+
+		desc = 'Yahoo Finance'
+		api_host = 'finance.yahoo.com'
+		ratelimit = 30
+		net_data_type = 'python'
+		has_verbose = False
+		asset_id_pat = r'^\^.*|.*=[xf]$'
+
+		@staticmethod
+		def get_id(sym,data):
+			return sym.lower()
+
+		@staticmethod
+		def conv_data(sym,data,btcusd):
+			price_usd = Decimal( data['regularMarketPrice']['raw'] )
+			return {
+				'id': sym,
+				'name': data['shortName'],
+				'symbol': sym.upper(),
+				'price_usd': str(price_usd),
+				'price_btc': str(price_usd / btcusd),
+				'percent_change_7d': None,
+				'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100,
+				'last_updated': data['regularMarketTime'],
+			}
+
+		def rate_limit_errmsg(self,elapsed):
+			return f'Rate limit exceeded!  Retry in {self.timeout-elapsed} seconds, or use --cached-data'
+
+		@property
+		def json_fn(self):
+			return os.path.join( cfg.cachedir, 'ticker-finance.json' )
+
+		@property
+		def timeout(self):
+			return 5 if gcfg.test_suite else self.ratelimit
+
+		def get_data_from_network(self):
+
+			arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi']
+
+			kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } }
+
+			if gcfg.test_suite:
+				kwargs.update({ 'timeout': 1, 'retry': 0 })
+
+			if gcfg.testing:
+				Msg('\nyahooquery.Ticker(\n  {},\n  {}\n)'.format(
+					arg,
+					fmt_dict(kwargs,fmt='kwargs') ))
+				return
+
+			from yahooquery import Ticker
+			return Ticker(arg,**kwargs).price
+
+		@staticmethod
+		def parse_asset_id(s,require_label):
+			return asset_tuple(
+				symbol = s.upper(),
+				id     = s.lower(),
+				source = 'fi' )
+
 def assets_list_gen(cfg_in):
 	for k,v in cfg_in.cfg['assets'].items():
 		yield('')
@@ -271,6 +335,23 @@ def gen_data(data):
 			btcusd = Decimal(d['price_usd'])
 			break
 
+	get_id = src_cls['fi'].get_id
+	conv_func = src_cls['fi'].conv_data
+
+	for k,v in data['fi'].items():
+		id = get_id(k,v)
+		if wants['id']:
+			if id in wants['id']:
+				if id in found['id']:
+					die(1,dup_sym_errmsg(id))
+				yield ( id, conv_func(id,v,btcusd) )
+				found['id'].add(id)
+				wants['id'].remove(id)
+				if id in usr_rate_assets_want['id']:
+					rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key
+		else:
+			break
+
 	for k in ('id','symbol'):
 		for d in data['cc']:
 			if wants[k]:
@@ -379,7 +460,7 @@ def make_cfg():
 		return tuple(gen())
 
 	def parse_asset_id(s,require_label=False):
-		return src_cls['cc'].parse_asset_id(s,require_label)
+		return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label)
 
 	def parse_usr_asset_arg(key,use_cf_file=False):
 		"""
@@ -489,11 +570,13 @@ def make_cfg():
 		'add_prec',
 		'cachedir',
 		'proxy',
+		'proxy2',
 		'portfolio' ])
 
 	global cfg_in,src_cls,cfg
 
 	src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() }
+	fi_pat = src_cls['fi'].asset_id_pat
 
 	cmd_args = gcfg._args
 	cfg_in = get_cfg_in()
@@ -514,6 +597,7 @@ def make_cfg():
 
 	proxy = get_proxy('proxy')
 	proxy = None if proxy == 'none' else proxy
+	proxy2 = get_proxy('proxy2')
 
 	cfg = cfg_tuple(
 		rows        = create_rows(),
@@ -526,6 +610,7 @@ def make_cfg():
 		add_prec    = parse_add_precision(gcfg.add_precision),
 		cachedir    = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir,
 		proxy       = proxy,
+		proxy2      = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy),
 		portfolio   = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None
 	)
 
@@ -541,9 +626,12 @@ def get_cfg_in():
 		cfg = cfg_data or {
 			'assets': {
 				'coin':      [ 'btc-bitcoin', 'eth-ethereum', 'xmr-monero' ],
-				'commodity': [ 'xau-gold-spot-token', 'xag-silver-spot-token', 'xbr-brent-crude-oil-spot' ],
-				'fiat':      [ 'gbp-pound-sterling-token', 'eur-euro-token' ],
-				'index':     [ 'dj30-dow-jones-30-token', 'spx-sp-500', 'ndx-nasdaq-100-token' ],
+				             # gold futures, silver futures, Brent futures
+				'commodity': [ 'gc=f', 'si=f', 'bz=f' ],
+				             # Pound Sterling, Euro, Swiss Franc
+				'fiat':      [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ],
+				             # Dow Jones Industrials, Nasdaq 100, S&P 500
+				'index':     [ '^dji', '^ixic', '^gspc' ],
 			},
 			'proxy': 'http://vpn-gw:8118'
 		},

mmgen_node_tools/data/ticker-cfg.yaml

@@ -3,10 +3,16 @@
 ### See the curl manpage for supported --proxy parameters
 ### For a direct connection, leave the right-hand side blank
 proxy: http://vpn-gw:8118
+# proxy2: http://gw2:8118
 
 ### Override the default cache directory (~/.cache/mmgen-node-tools):
 cachedir:
 
+### Additional asset columns:
+# add_columns:
+#  - cnhusd=x  # Yuan
+#  - 6j=f      # Yen futures
+
 ### Asset rows
 ### Asset labels are arbitrary strings.  Use as many or few as you wish.
 ### Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs.
@@ -19,13 +25,14 @@ assets:
     - ada-cardano
     - bnb-binance-coin
   commodity:
-    - xau-gold-spot-token
-    - xag-silver-spot-token
-    - xbr-brent-crude-oil-spot
+    - gc=f # gold futures
+    - si=f # silver futures
+    - bz=f # Brent futures
   fiat:
-    - gbp-pound-sterling-token
-    - eur-euro-token
+    - gbpusd=x # Pound Sterling
+    - eurusd=x # Euro
+    - chfusd=x # Swiss Franc
   index:
-    - dj30-dow-jones-30-token
-    - spx-sp-500
-    - ndx-nasdaq-100-token
+    - ^dji  # Dow Jones Industrials
+    - ^ixic # Nasdaq 100
+    - ^gspc # S&P 500

mmgen_node_tools/data/version

@@ -1 +1 @@
-3.2.dev0
+3.2.dev1

mmgen_node_tools/main_ticker.py

@@ -47,7 +47,7 @@ opts_data = {
 -r, --add-rows=LIST   Add rows for asset specifiers in LIST (comma-separated,
                       see ASSET SPECIFIERS below). Can also be used to supply
                       a USD exchange rate for missing assets.
--t, --testing         Print command to be executed to stdout and exit
+-t, --testing         Print command(s) to be executed to stdout and exit
 -T, --thousands-comma Use comma as a thousands separator
 -u, --update-time     Include UPDATED (last update time) column
 -v, --verbose         Be more verbose
@@ -55,6 +55,8 @@ opts_data = {
 -x, --proxy=P         Connect via proxy ‘P’.  Set to the empty string to
                       completely disable or ‘none’ to allow override from
                       environment. Consult the curl manpage for --proxy usage.
+-X, --proxy2=P        Alternate proxy for non-crypto financial data.  Defaults
+                      to value of --proxy
 """,
 	'notes': """
 
@@ -69,10 +71,15 @@ price to the spot price.
 
 ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids)
 consisting of symbol plus label (e.g. ‘xmr-monero’).  In cases where the
-symbol is ambiguous, the full ID must be used.  Examples:
+symbol is ambiguous, the full ID must be used.  For Yahoo Finance assets
+the symbol and ID are identical:
 
-  chf                   - specify asset by symbol
-  chf-swiss-franc-token - same as above, but use full ID instead of symbol
+Examples:
+
+  ltc           - specify asset by symbol
+  ltc-litecoin  - same as above, but use full ID instead of symbol
+  ^dji          - Dow Jones Industrial Average (Yahoo)
+  gc=f          - gold futures (Yahoo)
 
 ASSET SPECIFIERS have the following format:
 
@@ -89,7 +96,8 @@ postfixed with the letter ‘r’, its meaning is reversed, i.e. interpreted as
   inr:0.01257r           - same as above, but use reverse rate (INR/USD)
   inr-indian-rupee:79.5  - same as first example, but add an arbitrary label
   omr-omani-rial:2.59r   - Omani Rial is pegged to the Dollar at 2.59 USD
-  bgn-bg-lev:0.5113r:eur - Bulgarian Lev is pegged to the Euro at 0.5113 EUR
+  bgn-bulgarian-lev:0.5113r:eurusd=x
+                         - Bulgarian Lev is pegged to the Euro at 0.5113 EUR
 
 A TRADE_SPECIFIER is a single argument in the format:
 
@@ -99,8 +107,8 @@ A TRADE_SPECIFIER is a single argument in the format:
 
     xmr:17.34          - price of 17.34 XMR in all configured assets
     xmr-monero:17.34   - same as above, but with full ID
-    xmr:17.34:eur      - price of 17.34 XMR in EUR only
-    xmr:17.34:eur:2800 - commission on an offer of 17.34 XMR for 2800 EUR
+    xmr:17.34:eurusd=x - price of 17.34 XMR in EUR only
+    xmr:17.34:eurusd=x:2800 - commission on an offer of 17.34 XMR for 2800 EUR
 
   TO_AMOUNT, if included, is used to calculate the percentage difference or
   commission on an offer compared to the spot price.
@@ -112,28 +120,34 @@ A TRADE_SPECIFIER is a single argument in the format:
 
                                  PROXY NOTE
 
-The remote server used to obtain the price data, {cc.api_host!r}, blocks
-Tor behind a Captcha wall, so a Tor proxy cannot be used directly.  If you’re
-concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then
-set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’
-option in the config file or --proxy on the command line accordingly.  Or run
-the script directly on the VPN’ed host with ’proxy’ or --proxy set to the
-null string.
+The remote server used to obtain the crypto price data, {cc.api_host},
+blocks Tor behind a Captcha wall, so a Tor proxy cannot be used directly.
+If you’re concerned about privacy, connect via a VPN, or better yet, VPN over
+Tor.  Then set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the
+‘proxy’ option in the config file or --proxy on the command line accordingly.
+Or run the script directly on the VPN’ed host with ’proxy’ or --proxy set to
+the null string.
 
 Alternatively, you may download the JSON source data in a Tor-proxied browser
-from ‘{cc.api_url}’, save it as ‘ticker.json’ in your
-configured cache directory, and run the script with the --cached-data option.
+from {cc.api_url}, save it as ‘ticker.json’ in your
+configured cache directory and run the script with the --cached-data option.
+
+Financial data is obtained from {fi.desc}, which currently allows Tor.
 
 
                              RATE LIMITING NOTE
 
-To protect user privacy, all filtering and processing of data is performed
-client side so that the remote server does not know which assets are being
-examined.  This means that data for ALL available assets (currently over 4000)
-is fetched with each invocation of the script.  A rate limit of {cc.ratelimit} seconds
-between calls is thus imposed to prevent abuse of the remote server.  When the
---btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds.  To bypass the
-rate limit entirely, use --cached-data.
+To protect user privacy, all filtering and processing of cryptocurrency data
+is performed client side so that the remote server does not know which assets
+are being examined.  This means that data for ALL available crypto assets
+(currently over 8000) is fetched with each invocation of the script.  A rate
+limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the
+remote server.  When the --btc option is in effect, this limit is reduced to
+{cc.btc_ratelimit} seconds.  To bypass the rate limit entirely, use --cached-data.
+
+Note that financial data obtained from {fi.api_host} is filtered in the
+request, which has privacy implications.  The rate limit for financial data
+is {fi.ratelimit} seconds.
 
 
                                   EXAMPLES
@@ -146,10 +160,10 @@ $ mmnode-ticker --btc
 
 # Wide display, add EUR and OMR columns, OMR/USD rate, extra precision and
 # proxy:
-$ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118
+$ mmnode-ticker -w -c eurusd=x,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118
 
 # Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate:
-$ mmnode-ticker -wE -c eur,bgn-bulgarian-lev:0.5113r:eur
+$ mmnode-ticker -wE -c eurusd=x,bgn-bulgarian-lev:0.5113r:eurusd=x
 
 # Wide display, use cached data from previous network query, show portfolio
 # (see above), pipe output to pager, add DOGE row:
@@ -193,6 +207,7 @@ To add a portfolio, edit the file
 			cfg    = os.path.relpath(cfg_in.cfg_file,start=homedir),
 			pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir),
 			cc     = src_cls['cc'](),
+			fi     = src_cls['fi'](),
 		)
 	}
 }

setup.cfg

@@ -23,7 +23,8 @@ python_requires = >=3.7
 include_package_data = True
 
 install_requires =
-	mmgen>=13.3.dev44
+	mmgen>=14.0.dev2
+	yahooquery
 
 packages =
 	mmgen_node_tools

test/ref/ticker/ticker-cfg.yaml

@@ -12,13 +12,13 @@ assets:
     - ada-cardano
     - algo-algorand
   commodity:
-    - xau-gold-spot-token
-    - xag-silver-spot-token
-    - xbr-brent-crude-oil-spot
+    - gc=f # gold futures
+    - si=f # silver futures
+    - bz=f # Brent futures
   fiat:
-    - chf-swiss-franc-token
-    - eur-euro-token
+    - chfusd=x # Swiss Franc
+    - eurusd=x # Euro
   index:
-    - dj30-dow-jones-30-token
-    - spx-sp-500
-    - ndx-nasdaq-100-token
+    - ^dji  # Dow Jones Industrials
+    - ^ixic # Nasdaq 100
+    - ^gspc # S&P 500

test/test_py_d/ts_misc.py

@@ -87,7 +87,6 @@ class TestSuiteScripts(TestSuiteBase):
 		('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'),
 		('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'),
 		('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'),
-		('ticker18', 'ticker [--cached-data --wide --elapsed -c eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'),
 	)
 	}
 
@@ -102,6 +101,7 @@ class TestSuiteScripts(TestSuiteBase):
 	def ticker_setup(self):
 		self.spawn('',msg_only=True)
 		shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir)
+		shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir)
 		shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir)
 		return 'ok'
 
@@ -123,8 +123,8 @@ class TestSuiteScripts(TestSuiteBase):
 		if not cfg.skipping_deps:
 			t.expect('Creating')
 			t.expect('Creating')
-		t.expect('proxy host could not be resolved')
-		t.req_exit_val = 3
+		ret = t.expect(['proxy host could not be resolved','ProxyError'])
+		t.req_exit_val = 3 if ret == 0 else 1
 		return t
 
 	def ticker3(self):
@@ -138,15 +138,15 @@ class TestSuiteScripts(TestSuiteBase):
 
 	def ticker4(self):
 		return self.ticker(
-			['--wide','--add-columns=eur,inr-indian-rupee:79.5'],
+			['--wide','--add-columns=eurusd=x,inr-indian-rupee:79.5'],
 			[
-				r'EUR \(EURO TOKEN\) = 1.0186 USD ' +
+				r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' +
 				r'INR \(INDIAN RUPEE\) = 0.012579 USD',
-				'USD EUR INR BTC CHG_7d CHG_24h UPDATED',
+				'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED',
 				'BITCOIN',
-				r'ETHEREUM 1,659.66 1,629.3906 131,943.14 0.07146397 \+21.42 \+1.82',
-				r'MONERO 158.97 156.0734 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59',
-				r'INDIAN RUPEE 0.01 0.0123 1.00 0.00000054 -- --',
+				r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07146397 \+21.42 \+1.82',
+				r'MONERO 158.97 149.3870 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59',
+				r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --',
 			])
 
 	def ticker5(self):
@@ -213,7 +213,7 @@ class TestSuiteScripts(TestSuiteBase):
 				'SPOT PRICE',
 				'BTC 0.11783441',
 				'XMR 17.23400000',
-				'XAU','NDX',
+				'GC=F',r'\^IXIC',
 			])
 
 	def ticker11(self):
@@ -277,32 +277,22 @@ class TestSuiteScripts(TestSuiteBase):
 
 	def ticker16(self):
 		return self.ticker(
-			['--wide','--elapsed','-c','eur,omr-omani-rial:2.59r'],
+			['--wide','--elapsed','-c','eurusd=x,omr-omani-rial:2.59r'],
 			[
-				r'EUR \(EURO TOKEN\) = 1.0186 USD ' +
+				r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' +
 				r'OMR \(OMANI RIAL\) = 2.5900 USD',
-				'USD EUR OMR BTC CHG_7d CHG_24h UPDATED',
-				r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago',
-				'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- --'
+				'USD EURUSD=X OMR BTC CHG_7d CHG_24h UPDATED',
+				r'BITCOIN 23,250.77 21,848.7527 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago',
+				'OMANI RIAL 2.59 2.4338 1.0000 0.00011139 -- -- --'
 			])
 
 	def ticker17(self):
 		# BGN pegged at 0.5113 EUR
 		return self.ticker(
-			['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eur'],
+			['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eurusd=x'],
 			[
-				r'BGN \(BULGARIAN LEV\) = 0.52080 USD',
+				r'BGN \(BULGARIAN LEV\) = 0.54411 USD',
 				'USD BGN BTC CHG_7d CHG_24h UPDATED',
-				'BITCOIN 23,250.77 44,644.414 1.00000000',
-				'BULGARIAN LEV 0.52 1.000 0.00002240',
-			])
-
-	def ticker18(self):
-		return self.ticker(
-			['--wide','--elapsed','-c','eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'],
-			[
-				r'BGN \(BULGARIAN LEV\) = 0.52080 USD',
-				'USD EUR BGN BTC CHG_7d CHG_24h UPDATED',
-				'BITCOIN 23,250.77 22,826.6890 44,644.414 1.00000000',
-				'BULGARIAN LEV 0.52 0.5113 1.000 0.00002240',
+				'BITCOIN 23,250.77 42,731.767 1.00000000',
+				'BULGARIAN LEV 0.54 1.000 0.00002340',
 			])