mmgen/mmgen-node-tools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

mmnode-ticker: retrieve financial data from Yahoo Finance

Browse source
This commit is contained in:
The MMGen Project 2023-09-25 15:53:02 +00:00
commit fd318909c2
Signed by: mmgen
GPG key ID: 3F8B1861E32B7DA2
9 changed files with 179 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

96
mmgen_node_tools/Ticker.py
View file

@ -46,6 +46,7 @@ class DataSource:
sources = {
'cc': 'coinpaprika',
'fi': 'yahooquery'
}
class base:
@ -199,6 +200,69 @@ class DataSource:
id = (s.lower() if label else None),
source = 'cc' )
class yahooquery(base):
desc = 'Yahoo Finance'
api_host = 'finance.yahoo.com'
ratelimit = 30
net_data_type = 'python'
has_verbose = False
asset_id_pat = r'^\^.*|.*=[xf]$'
@staticmethod
def get_id(sym,data):
return sym.lower()
@staticmethod
def conv_data(sym,data,btcusd):
price_usd = Decimal( data['regularMarketPrice']['raw'] )
return {
'id': sym,
'name': data['shortName'],
'symbol': sym.upper(),
'price_usd': str(price_usd),
'price_btc': str(price_usd / btcusd),
'percent_change_7d': None,
'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100,
'last_updated': data['regularMarketTime'],
}
def rate_limit_errmsg(self,elapsed):
return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data'
@property
def json_fn(self):
return os.path.join( cfg.cachedir, 'ticker-finance.json' )
@property
def timeout(self):
return 5 if gcfg.test_suite else self.ratelimit
def get_data_from_network(self):
arg = [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi']
kwargs = { 'formatted': True, 'proxies': { 'https': cfg.proxy2 } }
if gcfg.test_suite:
kwargs.update({ 'timeout': 1, 'retry': 0 })
if gcfg.testing:
Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format(
arg,
fmt_dict(kwargs,fmt='kwargs') ))
return
from yahooquery import Ticker
return Ticker(arg,**kwargs).price
@staticmethod
def parse_asset_id(s,require_label):
return asset_tuple(
symbol = s.upper(),
id = s.lower(),
source = 'fi' )
def assets_list_gen(cfg_in):
for k,v in cfg_in.cfg['assets'].items():
yield('')
@ -271,6 +335,23 @@ def gen_data(data):
btcusd = Decimal(d['price_usd'])
break
get_id = src_cls['fi'].get_id
conv_func = src_cls['fi'].conv_data
for k,v in data['fi'].items():
id = get_id(k,v)
if wants['id']:
if id in wants['id']:
if id in found['id']:
die(1,dup_sym_errmsg(id))
yield ( id, conv_func(id,v,btcusd) )
found['id'].add(id)
wants['id'].remove(id)
if id in usr_rate_assets_want['id']:
rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key
else:
break
for k in ('id','symbol'):
for d in data['cc']:
if wants[k]:
@ -379,7 +460,7 @@ def make_cfg():
return tuple(gen())
def parse_asset_id(s,require_label=False):
return src_cls['cc'].parse_asset_id(s,require_label)
return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label)
def parse_usr_asset_arg(key,use_cf_file=False):
"""
@ -489,11 +570,13 @@ def make_cfg():
'add_prec',
'cachedir',
'proxy',
'proxy2',
'portfolio' ])
global cfg_in,src_cls,cfg
src_cls = { k: getattr(DataSource,v) for k,v in DataSource.sources.items() }
fi_pat = src_cls['fi'].asset_id_pat
cmd_args = gcfg._args
cfg_in = get_cfg_in()
@ -514,6 +597,7 @@ def make_cfg():
proxy = get_proxy('proxy')
proxy = None if proxy == 'none' else proxy
proxy2 = get_proxy('proxy2')
cfg = cfg_tuple(
rows = create_rows(),
@ -526,6 +610,7 @@ def make_cfg():
add_prec = parse_add_precision(gcfg.add_precision),
cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir,
proxy = proxy,
proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy),
portfolio = get_portfolio() if cfg_in.portfolio and gcfg.portfolio and not query else None
)
@ -541,9 +626,12 @@ def get_cfg_in():
cfg = cfg_data or {
'assets': {
'coin': [ 'btc-bitcoin', 'eth-ethereum', 'xmr-monero' ],
'commodity': [ 'xau-gold-spot-token', 'xag-silver-spot-token', 'xbr-brent-crude-oil-spot' ],
'fiat': [ 'gbp-pound-sterling-token', 'eur-euro-token' ],
'index': [ 'dj30-dow-jones-30-token', 'spx-sp-500', 'ndx-nasdaq-100-token' ],
# gold futures, silver futures, Brent futures
'commodity': [ 'gc=f', 'si=f', 'bz=f' ],
# Pound Sterling, Euro, Swiss Franc
'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ],
# Dow Jones Industrials, Nasdaq 100, S&P 500
'index': [ '^dji', '^ixic', '^gspc' ],
},
'proxy': 'http://vpn-gw:8118'
},

23
mmgen_node_tools/data/ticker-cfg.yaml
View file

@ -3,10 +3,16 @@
### See the curl manpage for supported --proxy parameters
### For a direct connection, leave the right-hand side blank
proxy: http://vpn-gw:8118
# proxy2: http://gw2:8118
### Override the default cache directory (~/.cache/mmgen-node-tools):
cachedir:
### Additional asset columns:
# add_columns:
# - cnhusd=x # Yuan
# - 6j=f # Yen futures
### Asset rows
### Asset labels are arbitrary strings. Use as many or few as you wish.
### Invoke ‘mmnode-ticker --list-ids’ for a full list of supported asset IDs.
@ -19,13 +25,14 @@ assets:
- ada-cardano
- bnb-binance-coin
commodity:
- xau-gold-spot-token
- xag-silver-spot-token
- xbr-brent-crude-oil-spot
- gc=f # gold futures
- si=f # silver futures
- bz=f # Brent futures
fiat:
- gbp-pound-sterling-token
- eur-euro-token
- gbpusd=x # Pound Sterling
- eurusd=x # Euro
- chfusd=x # Swiss Franc
index:
- dj30-dow-jones-30-token
- spx-sp-500
- ndx-nasdaq-100-token
- ^dji # Dow Jones Industrials
- ^ixic # Nasdaq 100
- ^gspc # S&P 500

2
mmgen_node_tools/data/version
View file

@ -1 +1 @@
3.2.dev0
3.2.dev1

65
mmgen_node_tools/main_ticker.py
View file

@ -47,7 +47,7 @@ opts_data = {
-r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated,
see ASSET SPECIFIERS below). Can also be used to supply
a USD exchange rate for missing assets.
-t, --testing Print command to be executed to stdout and exit
-t, --testing Print command(s) to be executed to stdout and exit
-T, --thousands-comma Use comma as a thousands separator
-u, --update-time Include UPDATED (last update time) column
-v, --verbose Be more verbose
@ -55,6 +55,8 @@ opts_data = {
-x, --proxy=P Connect via proxy ‘P’. Set to the empty string to
completely disable or ‘none’ to allow override from
environment. Consult the curl manpage for --proxy usage.
-X, --proxy2=P Alternate proxy for non-crypto financial data. Defaults
to value of --proxy
""",
'notes': """
@ -69,10 +71,15 @@ price to the spot price.
ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids)
consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the
symbol is ambiguous, the full ID must be used. Examples:
symbol is ambiguous, the full ID must be used. For Yahoo Finance assets
the symbol and ID are identical:
chf - specify asset by symbol
chf-swiss-franc-token - same as above, but use full ID instead of symbol
Examples:
ltc - specify asset by symbol
ltc-litecoin - same as above, but use full ID instead of symbol
^dji - Dow Jones Industrial Average (Yahoo)
gc=f - gold futures (Yahoo)
ASSET SPECIFIERS have the following format:
@ -89,7 +96,8 @@ postfixed with the letter ‘r’, its meaning is reversed, i.e. interpreted as
inr:0.01257r - same as above, but use reverse rate (INR/USD)
inr-indian-rupee:79.5 - same as first example, but add an arbitrary label
omr-omani-rial:2.59r - Omani Rial is pegged to the Dollar at 2.59 USD
bgn-bg-lev:0.5113r:eur - Bulgarian Lev is pegged to the Euro at 0.5113 EUR
bgn-bulgarian-lev:0.5113r:eurusd=x
- Bulgarian Lev is pegged to the Euro at 0.5113 EUR
A TRADE_SPECIFIER is a single argument in the format:
@ -99,8 +107,8 @@ A TRADE_SPECIFIER is a single argument in the format:
xmr:17.34 - price of 17.34 XMR in all configured assets
xmr-monero:17.34 - same as above, but with full ID
xmr:17.34:eur - price of 17.34 XMR in EUR only
xmr:17.34:eur:2800 - commission on an offer of 17.34 XMR for 2800 EUR
xmr:17.34:eurusd=x - price of 17.34 XMR in EUR only
xmr:17.34:eurusd=x:2800 - commission on an offer of 17.34 XMR for 2800 EUR
TO_AMOUNT, if included, is used to calculate the percentage difference or
commission on an offer compared to the spot price.
@ -112,28 +120,34 @@ A TRADE_SPECIFIER is a single argument in the format:
PROXY NOTE
The remote server used to obtain the price data, {cc.api_host!r}, blocks
Tor behind a Captcha wall, so a Tor proxy cannot be used directly. If you’re
concerned about privacy, connect via a VPN, or better yet, VPN over Tor. Then
set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the ‘proxy’
option in the config file or --proxy on the command line accordingly. Or run
the script directly on the VPN’ed host with ’proxy’ or --proxy set to the
null string.
The remote server used to obtain the crypto price data, {cc.api_host},
blocks Tor behind a Captcha wall, so a Tor proxy cannot be used directly.
If you’re concerned about privacy, connect via a VPN, or better yet, VPN over
Tor. Then set up an HTTP proxy (e.g. Privoxy) on the VPN’ed host and set the
‘proxy’ option in the config file or --proxy on the command line accordingly.
Or run the script directly on the VPN’ed host with ’proxy’ or --proxy set to
the null string.
Alternatively, you may download the JSON source data in a Tor-proxied browser
from ‘{cc.api_url}’, save it as ‘ticker.json’ in your
configured cache directory, and run the script with the --cached-data option.
from {cc.api_url}, save it as ‘ticker.json’ in your
configured cache directory and run the script with the --cached-data option.
Financial data is obtained from {fi.desc}, which currently allows Tor.
RATE LIMITING NOTE
To protect user privacy, all filtering and processing of data is performed
client side so that the remote server does not know which assets are being
examined. This means that data for ALL available assets (currently over 4000)
is fetched with each invocation of the script. A rate limit of {cc.ratelimit} seconds
between calls is thus imposed to prevent abuse of the remote server. When the
--btc option is in effect, this limit is reduced to {cc.btc_ratelimit} seconds. To bypass the
rate limit entirely, use --cached-data.
To protect user privacy, all filtering and processing of cryptocurrency data
is performed client side so that the remote server does not know which assets
are being examined. This means that data for ALL available crypto assets
(currently over 8000) is fetched with each invocation of the script. A rate
limit of {cc.ratelimit} seconds between calls is thus imposed to prevent abuse of the
remote server. When the --btc option is in effect, this limit is reduced to
{cc.btc_ratelimit} seconds. To bypass the rate limit entirely, use --cached-data.
Note that financial data obtained from {fi.api_host} is filtered in the
request, which has privacy implications. The rate limit for financial data
is {fi.ratelimit} seconds.
EXAMPLES
@ -146,10 +160,10 @@ $ mmnode-ticker --btc
# Wide display, add EUR and OMR columns, OMR/USD rate, extra precision and
# proxy:
$ mmnode-ticker -w -c eur,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118
$ mmnode-ticker -w -c eurusd=x,omr-omani-rial:2.59r -e2 -x http://vpnhost:8118
# Wide display, elapsed update time, add EUR, BGN columns and BGN/EUR rate:
$ mmnode-ticker -wE -c eur,bgn-bulgarian-lev:0.5113r:eur
$ mmnode-ticker -wE -c eurusd=x,bgn-bulgarian-lev:0.5113r:eurusd=x
# Wide display, use cached data from previous network query, show portfolio
# (see above), pipe output to pager, add DOGE row:
@ -193,6 +207,7 @@ To add a portfolio, edit the file
cfg = os.path.relpath(cfg_in.cfg_file,start=homedir),
pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir),
cc = src_cls['cc'](),
fi = src_cls['fi'](),
)
}
}

3
setup.cfg
View file

@ -23,7 +23,8 @@ python_requires = >=3.7
include_package_data = True
install_requires =
mmgen>=13.3.dev44
mmgen>=14.0.dev2
yahooquery
packages =
mmgen_node_tools

16
test/ref/ticker/ticker-cfg.yaml
View file

@ -12,13 +12,13 @@ assets:
- ada-cardano
- algo-algorand
commodity:
- xau-gold-spot-token
- xag-silver-spot-token
- xbr-brent-crude-oil-spot
- gc=f # gold futures
- si=f # silver futures
- bz=f # Brent futures
fiat:
- chf-swiss-franc-token
- eur-euro-token
- chfusd=x # Swiss Franc
- eurusd=x # Euro
index:
- dj30-dow-jones-30-token
- spx-sp-500
- ndx-nasdaq-100-token
- ^dji # Dow Jones Industrials
- ^ixic # Nasdaq 100
- ^gspc # S&P 500

1
test/ref/ticker/ticker-finance.json Normal file
View file

File diff suppressed because one or more lines are too long

2
test/ref/ticker/ticker.json
View file

File diff suppressed because one or more lines are too long

48
test/test_py_d/ts_misc.py
View file

@ -87,7 +87,6 @@ class TestSuiteScripts(TestSuiteBase):
('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'),
('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'),
('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'),
('ticker18', 'ticker [--cached-data --wide --elapsed -c eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'),
)
}
@ -102,6 +101,7 @@ class TestSuiteScripts(TestSuiteBase):
def ticker_setup(self):
self.spawn('',msg_only=True)
shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir)
shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir)
shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir)
return 'ok'
@ -123,8 +123,8 @@ class TestSuiteScripts(TestSuiteBase):
if not cfg.skipping_deps:
t.expect('Creating')
t.expect('Creating')
t.expect('proxy host could not be resolved')
t.req_exit_val = 3
ret = t.expect(['proxy host could not be resolved','ProxyError'])
t.req_exit_val = 3 if ret == 0 else 1
return t
def ticker3(self):
@ -138,15 +138,15 @@ class TestSuiteScripts(TestSuiteBase):
def ticker4(self):
return self.ticker(
['--wide','--add-columns=eur,inr-indian-rupee:79.5'],
['--wide','--add-columns=eurusd=x,inr-indian-rupee:79.5'],
[
r'EUR \(EURO TOKEN\) = 1.0186 USD ' +
r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' +
r'INR \(INDIAN RUPEE\) = 0.012579 USD',
'USD EUR INR BTC CHG_7d CHG_24h UPDATED',
'USD EURUSD=X INR BTC CHG_7d CHG_24h UPDATED',
'BITCOIN',
r'ETHEREUM 1,659.66 1,629.3906 131,943.14 0.07146397 \+21.42 \+1.82',
r'MONERO 158.97 156.0734 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59',
r'INDIAN RUPEE 0.01 0.0123 1.00 0.00000054 -- --',
r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07146397 \+21.42 \+1.82',
r'MONERO 158.97 149.3870 12,638.36 0.00684527 \+7.28 \+1.21 2022-08-02 18:25:59',
r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- --',
])
def ticker5(self):
@ -213,7 +213,7 @@ class TestSuiteScripts(TestSuiteBase):
'SPOT PRICE',
'BTC 0.11783441',
'XMR 17.23400000',
'XAU','NDX',
'GC=F',r'\^IXIC',
])
def ticker11(self):
@ -277,32 +277,22 @@ class TestSuiteScripts(TestSuiteBase):
def ticker16(self):
return self.ticker(
['--wide','--elapsed','-c','eur,omr-omani-rial:2.59r'],
['--wide','--elapsed','-c','eurusd=x,omr-omani-rial:2.59r'],
[
r'EUR \(EURO TOKEN\) = 1.0186 USD ' +
r'EURUSD=X \(EUR/USD\) = 1.0642 USD ' +
r'OMR \(OMANI RIAL\) = 2.5900 USD',
'USD EUR OMR BTC CHG_7d CHG_24h UPDATED',
r'BITCOIN 23,250.77 22,826.6890 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago',
'OMANI RIAL 2.59 2.5428 1.0000 0.00011139 -- -- --'
'USD EURUSD=X OMR BTC CHG_7d CHG_24h UPDATED',
r'BITCOIN 23,250.77 21,848.7527 8,977.1328 1.00000000 \+11.15 \+0.89 10 minutes ago',
'OMANI RIAL 2.59 2.4338 1.0000 0.00011139 -- -- --'
])
def ticker17(self):
# BGN pegged at 0.5113 EUR
return self.ticker(
['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eur'],
['--wide','--elapsed','-c','bgn-bulgarian-lev:0.5113r:eurusd=x'],
[
r'BGN \(BULGARIAN LEV\) = 0.52080 USD',
r'BGN \(BULGARIAN LEV\) = 0.54411 USD',
'USD BGN BTC CHG_7d CHG_24h UPDATED',
'BITCOIN 23,250.77 44,644.414 1.00000000',
'BULGARIAN LEV 0.52 1.000 0.00002240',
])
def ticker18(self):
return self.ticker(
['--wide','--elapsed','-c','eur,bgn-bulgarian-lev:0.5113r:eur-euro-token'],
[
r'BGN \(BULGARIAN LEV\) = 0.52080 USD',
'USD EUR BGN BTC CHG_7d CHG_24h UPDATED',
'BITCOIN 23,250.77 22,826.6890 44,644.414 1.00000000',
'BULGARIAN LEV 0.52 0.5113 1.000 0.00002240',
'BITCOIN 23,250.77 42,731.767 1.00000000',
'BULGARIAN LEV 0.54 1.000 0.00002340',
])