From af7c14fe369fc37d4d465e9a4bcb9f35b1952173 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 5 Nov 2023 13:40:23 +0000 Subject: [PATCH] mmnode-ticker: get historical financial data from Yahoo Finance --- mmgen_node_tools/Ticker.py | 43 ++++++++++++++++++++- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 3 +- test/ref/ticker/ticker-finance-history.json | 1 + 5 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 test/ref/ticker/ticker-finance-history.json diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 4011c90..6c15906 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -48,6 +48,7 @@ class DataSource: 'cc': 'coinpaprika' }, { 'fi': 'yahoospot', + 'hi': 'yahoohist', } ] @@ -248,7 +249,9 @@ class DataSource: 'symbol': sym.upper(), 'price_usd': price_usd, 'price_btc': price_usd / btcusd, - 'percent_change_7d': None, + 'percent_change_1y': data['pct_chg_1y'], + 'percent_change_30d': data['pct_chg_4wks'], + 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, 'last_updated': data['regularMarketTime'], } @@ -272,6 +275,7 @@ class DataSource: kwargs = { 'formatted': True, + 'asynchronous': True, 'proxies': { 'https': cfg.proxy2 }, } @@ -297,6 +301,33 @@ class DataSource: id = s.lower(), source = 'fi' ) + class yahoohist(yahoospot): + + json_fn_basename = 'ticker-finance-history.json' + data_desc = 'historical financial data' + net_data_type = 'json' + period = '1y' + interval = '1wk' + + def process_network_data(self,ticker): + return ticker.history( + period = self.period, + interval = self.interval).to_json(orient='index') + + def postprocess_data(self,data): + def gen(): + keys = set() + for key,val in data.items(): + if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): + date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) + if (sym := m[1]) in keys: + d[date] = val + else: + keys.add(sym) + d = {date:val} + yield (sym,d) + return dict(gen()) + def assets_list_gen(cfg_in): for k,v in cfg_in.cfg['assets'].items(): yield '' @@ -378,6 +409,16 @@ def gen_data(data): if id in wants['id']: if id in found['id']: die(1,dup_sym_errmsg(id)) + if m := data['hi'].get(k): + spot = v['regularMarketPrice']['raw'] + hist = tuple(m.values()) + v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( + (spot / hist[-2]['close'] - 1) * 100, + (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month + (spot / hist[0]['close'] - 1) * 100, + ) + else: + v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None yield ( id, conv_func(id,v,btcusd) ) found['id'].add(id) wants['id'].remove(id) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 5223fbd..ce7b703 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.2.dev3 +3.2.dev4 diff --git a/setup.cfg b/setup.cfg index 0c1e362..339e7f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ python_requires = >=3.8 include_package_data = True install_requires = - mmgen>=14.0.dev9 + mmgen>=14.0.dev11 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 8046e51..3dbac50 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -102,6 +102,7 @@ class CmdTestScripts(CmdTestBase): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) + shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' @@ -146,7 +147,7 @@ class CmdTestScripts(CmdTestBase): 'BITCOIN', r'ETHEREUM 1,659.66 1,559.5846 131,943.14 0.07138094 \+36.41 \+29.99 \+21.42 \+1.82', r'MONERO 158.97 149.3870 12,638.36 0.00683732 \+12.38 \+10.19 \+7.28 \+1.21 2022-08-02 18:25:59', - r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -- -- -- -0.23', + r'S&P 500 4,320.06 4,059.5604 343,444.77 0.18580285 -1.71 \+12.93 \+9.05 -0.23', r'INDIAN RUPEE 0.01 0.0118 1.00 0.00000054 -- -- -- --', ]) diff --git a/test/ref/ticker/ticker-finance-history.json b/test/ref/ticker/ticker-finance-history.json new file mode 100644 index 0000000..e8b0bd3 --- /dev/null +++ b/test/ref/ticker/ticker-finance-history.json @@ -0,0 +1 @@ +{"('BZ=F', datetime.date(2021, 8, 2))":{"open":75.1800003052,"high":75.3300018311,"low":69.75,"close":70.6999969482,"volume":192386,"adjclose":70.6999969482},"('BZ=F', datetime.date(2022, 6, 27))":{"open":112.4599990845,"high":120.3799972534,"low":108.0100021362,"close":111.6299972534,"volume":64094,"adjclose":111.6299972534},"('BZ=F', datetime.date(2022, 7, 4))":{"open":111.6100006104,"high":114.6900024414,"low":98.4700012207,"close":107.0199966431,"volume":115623,"adjclose":107.0199966431},"('BZ=F', datetime.date(2022, 7, 11))":{"open":106.7600021362,"high":107.6600036621,"low":94.5,"close":101.1600036621,"volume":109537,"adjclose":101.1600036621},"('BZ=F', datetime.date(2022, 7, 18))":{"open":100.9199981689,"high":107.5999984741,"low":99.4800033569,"close":103.1999969482,"volume":110290,"adjclose":103.1999969482},"('BZ=F', datetime.date(2022, 7, 25))":{"open":103.4400024414,"high":110.4300003052,"low":101.6699981689,"close":110.0100021362,"volume":77572,"adjclose":110.0100021362},"('BZ=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":102.4400024414,"high":111.4300003052,"low":101.6699981689,"close":111.0100021362,"volume":77572,"adjclose":111.0100021362},"('CHFUSD=X', datetime.date(2021, 7, 26))":{"open":1.0939244032,"high":1.1064150333,"low":1.0908932686,"close":1.1047636271,"volume":0,"adjclose":1.1047636271},"('CHFUSD=X', datetime.date(2022, 7, 4))":{"open":1.043405652,"high":1.0446264744,"low":1.020783186,"close":1.0241703987,"volume":0,"adjclose":1.0241703987},"('CHFUSD=X', datetime.date(2022, 7, 11))":{"open":1.022777319,"high":1.0247056484,"low":1.0118077993,"close":1.0238809586,"volume":0,"adjclose":1.0238809586},"('CHFUSD=X', datetime.date(2022, 7, 18))":{"open":1.0250102282,"high":1.0417101383,"low":1.0216904879,"close":1.0396090746,"volume":0,"adjclose":1.0396090746},"('CHFUSD=X', datetime.date(2022, 7, 25))":{"open":1.0384216309,"high":1.0522992611,"low":1.0345113277,"close":1.0506275892,"volume":0,"adjclose":1.0506275892},"('CHFUSD=X', datetime.date(2022, 8, 1))":{"open":1.0492409468,"high":1.0547411442,"low":1.0495601892,"close":1.0492409468,"volume":0,"adjclose":1.0492409468},"('CHFUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.0492409468,"high":1.0547411442,"low":1.0495601892,"close":1.0492409468,"volume":0,"adjclose":1.0492409468},"('EURUSD=X', datetime.date(2021, 7, 26))":{"open":1.1822398901,"high":1.1911001205,"low":1.1785781384,"close":1.1867604256,"volume":0,"adjclose":1.1867604256},"('EURUSD=X', datetime.date(2022, 7, 4))":{"open":1.043394804,"high":1.0463534594,"low":1.0079730749,"close":1.0187449455,"volume":0,"adjclose":1.0187449455},"('EURUSD=X', datetime.date(2022, 7, 11))":{"open":1.0166114569,"high":1.0167768002,"low":0.9953616261,"close":1.008867979,"volume":0,"adjclose":1.008867979},"('EURUSD=X', datetime.date(2022, 7, 18))":{"open":1.0096318722,"high":1.0275379419,"low":1.0082373619,"close":1.0215548277,"volume":0,"adjclose":1.0215548277},"('EURUSD=X', datetime.date(2022, 7, 25))":{"open":1.0200231075,"high":1.0256409645,"low":1.0107442141,"close":1.0227040052,"volume":0,"adjclose":1.0227040052},"('EURUSD=X', datetime.date(2022, 8, 1))":{"open":1.02082479,"high":1.0277491808,"low":1.020960331,"close":1.02082479,"volume":0,"adjclose":1.02082479},"('EURUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.02082479,"high":1.0277491808,"low":1.020960331,"close":1.02082479,"volume":0,"adjclose":1.02082479},"('GBPUSD=X', datetime.date(2021, 7, 26))":{"open":1.3886767626,"high":1.3984057903,"low":1.3846387863,"close":1.3910005093,"volume":0,"adjclose":1.3910005093},"('GBPUSD=X', datetime.date(2022, 7, 4))":{"open":1.2105804682,"high":1.2165154219,"low":1.1877613068,"close":1.2031002045,"volume":0,"adjclose":1.2031002045},"('GBPUSD=X', datetime.date(2022, 7, 11))":{"open":1.2017786503,"high":1.2019952536,"low":1.1763184071,"close":1.1855996847,"volume":0,"adjclose":1.1855996847},"('GBPUSD=X', datetime.date(2022, 7, 18))":{"open":1.1887921095,"high":1.2064181566,"low":1.1875497103,"close":1.2005999088,"volume":0,"adjclose":1.2005999088},"('GBPUSD=X', datetime.date(2022, 7, 25))":{"open":1.1984229088,"high":1.2242598534,"low":1.1962151527,"close":1.2180001736,"volume":0,"adjclose":1.2180001736},"('GBPUSD=X', datetime.date(2022, 8, 1))":{"open":1.2167818546,"high":1.2291958332,"low":1.2164857388,"close":1.2167373896,"volume":0,"adjclose":1.2167373896},"('GBPUSD=X', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1.2167818546,"high":1.2291958332,"low":1.2164857388,"close":1.2167373896,"volume":0,"adjclose":1.2167373896},"('GC=F', datetime.date(2021, 7, 26))":{"open":1803.6999511719,"high":1832.5999755859,"low":1799.5,"close":1812.5999755859,"volume":243380,"adjclose":1812.5999755859},"('GC=F', datetime.date(2022, 6, 27))":{"open":1830.5,"high":1830.6999511719,"low":1791.5999755859,"close":1798.9000244141,"volume":1046,"adjclose":1798.9000244141},"('GC=F', datetime.date(2022, 7, 4))":{"open":1805.4000244141,"high":1805.4000244141,"low":1732.0999755859,"close":1740.5999755859,"volume":4032,"adjclose":1740.5999755859},"('GC=F', datetime.date(2022, 7, 11))":{"open":1732.5,"high":1736.6999511719,"low":1701.0999755859,"close":1702.4000244141,"volume":2904,"adjclose":1702.4000244141},"('GC=F', datetime.date(2022, 7, 18))":{"open":1712.1999511719,"high":1735,"low":1679.8000488281,"close":1727.0999755859,"volume":2322,"adjclose":1727.0999755859},"('GC=F', datetime.date(2022, 7, 25))":{"open":1727,"high":1765.6999511719,"low":1717.6999511719,"close":1762.9000244141,"volume":185654,"adjclose":1762.9000244141},"('GC=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":1727,"high":1765.6999511719,"low":1717.6999511719,"close":1762.9000244141,"volume":185654,"adjclose":1762.9000244141},"('SI=F', datetime.date(2021, 7, 26))":{"open":25.2000007629,"high":25.8549995422,"low":24.6319999695,"close":25.5279998779,"volume":1000,"adjclose":25.5279998779},"('SI=F', datetime.date(2022, 6, 27))":{"open":21.2950000763,"high":21.3299999237,"low":19.2649993896,"close":19.5970001221,"volume":48988,"adjclose":19.5970001221},"('SI=F', datetime.date(2022, 7, 4))":{"open":19.8099994659,"high":20.0699996948,"low":18.8400001526,"close":19.1669998169,"volume":566,"adjclose":19.1669998169},"('SI=F', datetime.date(2022, 7, 11))":{"open":19.1650009155,"high":19.1749992371,"low":18,"close":18.5480003357,"volume":1108,"adjclose":18.5480003357},"('SI=F', datetime.date(2022, 7, 18))":{"open":18.8099994659,"high":18.9400005341,"low":18.1100006104,"close":18.5849990845,"volume":690,"adjclose":18.5849990845},"('SI=F', datetime.date(2022, 7, 25))":{"open":18.4099998474,"high":20.2900009155,"low":18.1849994659,"close":20.1560001373,"volume":980,"adjclose":20.1560001373},"('SI=F', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":18.4099998474,"high":20.2900009155,"low":18.1849994659,"close":20.1560001373,"volume":980,"adjclose":20.1560001373},"('^DJI', datetime.date(2021, 7, 26))":{"open":35078.8984375,"high":35171.51953125,"low":34871.12890625,"close":34935.46875,"volume":1172870000,"adjclose":34935.46875},"('^DJI', datetime.date(2022, 6, 27))":{"open":31533.599609375,"high":31885.08984375,"low":30431.869140625,"close":31097.259765625,"volume":1634590000,"adjclose":31097.259765625},"('^DJI', datetime.date(2022, 7, 4))":{"open":30903.119140625,"high":31511.4609375,"low":30355.119140625,"close":31338.150390625,"volume":1147620000,"adjclose":31338.150390625},"('^DJI', datetime.date(2022, 7, 11))":{"open":31277.98046875,"high":31367.55078125,"low":30143.9296875,"close":31288.259765625,"volume":1507740000,"adjclose":31288.259765625},"('^DJI', datetime.date(2022, 7, 18))":{"open":31475.98046875,"high":32219.25,"low":30982.970703125,"close":31899.2890625,"volume":1624400000,"adjclose":31899.2890625},"('^DJI', datetime.date(2022, 7, 25))":{"open":31950.9296875,"high":32910.1796875,"low":31705.359375,"close":32845.12890625,"volume":1764330000,"adjclose":32845.12890625},"('^DJI', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":31950.9296875,"high":32910.1796875,"low":31705.359375,"close":32845.12890625,"volume":1764330000,"adjclose":32845.12890625},"('^GSPC', datetime.date(2021, 7, 26))":{"open":4416.3798828125,"high":4429.9702148438,"low":4372.509765625,"close":4395.259765625,"volume":16458580000,"adjclose":4395.259765625},"('^GSPC', datetime.date(2022, 6, 27))":{"open":3920.7600097656,"high":3945.8601074219,"low":3738.669921875,"close":3825.330078125,"volume":21693690000,"adjclose":3825.330078125},"('^GSPC', datetime.date(2022, 7, 4))":{"open":3792.6101074219,"high":3918.5,"low":3742.0600585938,"close":3899.3798828125,"volume":17073700000,"adjclose":3899.3798828125},"('^GSPC', datetime.date(2022, 7, 11))":{"open":3880.9399414062,"high":3880.9399414062,"low":3721.5600585938,"close":3863.1599121094,"volume":19693570000,"adjclose":3863.1599121094},"('^GSPC', datetime.date(2022, 7, 18))":{"open":3883.7900390625,"high":4012.4399414062,"low":3818.6298828125,"close":3961.6298828125,"volume":20385270000,"adjclose":3961.6298828125},"('^GSPC', datetime.date(2022, 7, 25))":{"open":3965.7199707031,"high":4140.1499023438,"low":3910.7399902344,"close":4130.2900390625,"volume":20488830000,"adjclose":4130.2900390625},"('^GSPC', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":3965.7199707031,"high":4140.1499023438,"low":3910.7399902344,"close":4130.2900390625,"volume":20488830000,"adjclose":4130.2900390625},"('^IXIC', datetime.date(2021, 7, 26))":{"open":14807.9501953125,"high":14833.740234375,"low":14503.759765625,"close":14672.6796875,"volume":16171130000,"adjclose":14672.6796875},"('^IXIC', datetime.date(2022, 6, 27))":{"open":11661.01953125,"high":11677.490234375,"low":10850.009765625,"close":11127.849609375,"volume":26652190000,"adjclose":11127.849609375},"('^IXIC', datetime.date(2022, 7, 4))":{"open":10964.1796875,"high":11689.7001953125,"low":10911.4501953125,"close":11635.3095703125,"volume":19116360000,"adjclose":11635.3095703125},"('^IXIC', datetime.date(2022, 7, 11))":{"open":11524.490234375,"high":11541.099609375,"low":11005.9296875,"close":11452.419921875,"volume":21963960000,"adjclose":11452.419921875},"('^IXIC', datetime.date(2022, 7, 18))":{"open":11561.6396484375,"high":12093.01953125,"low":11322.83984375,"close":11834.1103515625,"volume":25230550000,"adjclose":11834.1103515625},"('^IXIC', datetime.date(2022, 7, 25))":{"open":11837.9599609375,"high":12426.259765625,"low":11533.3701171875,"close":12390.6904296875,"volume":23117120000,"adjclose":12390.6904296875},"('^IXIC', datetime.datetime(2022, 8, 2, 7, 7, 7, tzinfo=))":{"open":11837.9599609375,"high":12426.259765625,"low":11533.3701171875,"close":12390.6904296875,"volume":23117120000,"adjclose":12390.6904296875}}