From 61c4696c6533e9dc5360f09d2b23766777a4d202 Mon Sep 17 00:00:00 2001 From: Value Raider Date: Sat, 6 Jan 2024 23:41:50 +0000 Subject: [PATCH] Add new data to README, remove deprecated stuff, fix tests, v0.2.35 Ticker.recommendations*: - add to README - organise their unit tests - remove redundant recommendations_history Remove deprecated arguments from Ticker.history Fix 'bad symbol' behaviour & tests Fix some prices tests Bump version 0.2.35 --- CHANGELOG.rst | 4 ++ README.md | 5 ++ meta.yaml | 2 +- tests/prices.py | 89 ++++-------------------------- tests/ticker.py | 102 +++++++++++++++++++++-------------- yfinance/base.py | 15 ------ yfinance/scrapers/holders.py | 20 ++++++- yfinance/scrapers/quote.py | 53 ++++++++++++------ yfinance/ticker.py | 4 -- yfinance/version.py | 2 +- 10 files changed, 138 insertions(+), 158 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6565ca051..168873f9a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ Change Log =========== +0.2.35 +------ +Internal fixes for 0.2.34 + 0.2.34 ------ Features: diff --git a/README.md b/README.md index ac4bd5bae..356a28416 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,11 @@ msft.insider_transactions msft.insider_purchases msft.insider_roster_holders +# show recommendations +msft.recommendations +msft.recommendations_summary +msft.upgrades_downgrades + # Show future and historic earnings dates, returns at most next 4 quarters and last 8 quarters by default. # Note: If more are needed use msft.get_earnings_dates(limit=XX) with increased limit argument. msft.earnings_dates diff --git a/meta.yaml b/meta.yaml index 5d819cf29..843a8f43f 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,5 +1,5 @@ {% set name = "yfinance" %} -{% set version = "0.2.34" %} +{% set version = "0.2.35" %} package: name: "{{ name|lower }}" diff --git a/tests/prices.py b/tests/prices.py index cb61bcde6..c0fa4df0a 100644 --- a/tests/prices.py +++ b/tests/prices.py @@ -399,71 +399,20 @@ def test_dst_fix(self): raise def test_prune_post_intraday_us(self): - # Half-day before USA Thanksgiving. Yahoo normally + # Half-day at USA Thanksgiving. Yahoo normally # returns an interval starting when regular trading closes, # even if prepost=False. # Setup tkr = "AMZN" - interval = "1h" - interval_td = _dt.timedelta(hours=1) - time_open = _dt.time(9, 30) - time_close = _dt.time(16) - special_day = _dt.date(2022, 11, 25) + special_day = _dt.date(2023, 11, 24) time_early_close = _dt.time(13) dat = yf.Ticker(tkr, session=self.session) # Run start_d = special_day - _dt.timedelta(days=7) end_d = special_day + _dt.timedelta(days=7) - df = dat.history(start=start_d, end=end_d, interval=interval, prepost=False, keepna=True) - tg_last_dt = df.loc[str(special_day)].index[-1] - self.assertTrue(tg_last_dt.time() < time_early_close) - - # Test no other afternoons (or mornings) were pruned - start_d = _dt.date(special_day.year, 1, 1) - end_d = _dt.date(special_day.year+1, 1, 1) df = dat.history(start=start_d, end=end_d, interval="1h", prepost=False, keepna=True) - last_dts = _pd.Series(df.index).groupby(df.index.date).last() - f_early_close = (last_dts+interval_td).dt.time < time_close - early_close_dates = last_dts.index[f_early_close].values - self.assertEqual(len(early_close_dates), 1) - self.assertEqual(early_close_dates[0], special_day) - - first_dts = _pd.Series(df.index).groupby(df.index.date).first() - f_late_open = first_dts.dt.time > time_open - late_open_dates = first_dts.index[f_late_open] - self.assertEqual(len(late_open_dates), 0) - - def test_prune_post_intraday_omx(self): - # Half-day before Sweden Christmas. Yahoo normally - # returns an interval starting when regular trading closes, - # even if prepost=False. - # If prepost=False, test that yfinance is removing prepost intervals. - - # Setup - tkr = "AEC.ST" - interval = "1h" - interval_td = _dt.timedelta(hours=1) - time_open = _dt.time(9) - time_close = _dt.time(17, 30) - special_day = _dt.date(2022, 12, 23) - time_early_close = _dt.time(13, 2) - dat = yf.Ticker(tkr, session=self.session) - - # Half trading day Jan 5, Apr 14, May 25, Jun 23, Nov 4, Dec 23, Dec 30 - half_days = [_dt.date(special_day.year, x[0], x[1]) for x in [(1, 5), (4, 14), (5, 25), (6, 23), (11, 4), (12, 23), (12, 30)]] - - # Yahoo has incorrectly classified afternoon of 2022-04-13 as post-market. - # Nothing yfinance can do because Yahoo doesn't return data with prepost=False. - # But need to handle in this test. - expected_incorrect_half_days = [_dt.date(2022, 4, 13)] - half_days = sorted(half_days+expected_incorrect_half_days) - - # Run - start_d = special_day - _dt.timedelta(days=7) - end_d = special_day + _dt.timedelta(days=7) - df = dat.history(start=start_d, end=end_d, interval=interval, prepost=False, keepna=True) tg_last_dt = df.loc[str(special_day)].index[-1] self.assertTrue(tg_last_dt.time() < time_early_close) @@ -472,40 +421,22 @@ def test_prune_post_intraday_omx(self): end_d = _dt.date(special_day.year+1, 1, 1) df = dat.history(start=start_d, end=end_d, interval="1h", prepost=False, keepna=True) last_dts = _pd.Series(df.index).groupby(df.index.date).last() - f_early_close = (last_dts+interval_td).dt.time < time_close - early_close_dates = last_dts.index[f_early_close].values - unexpected_early_close_dates = [d for d in early_close_dates if d not in half_days] - self.assertEqual(len(unexpected_early_close_dates), 0) - self.assertEqual(len(early_close_dates), len(half_days)) - self.assertTrue(_np.equal(early_close_dates, half_days).all()) - - first_dts = _pd.Series(df.index).groupby(df.index.date).first() - f_late_open = first_dts.dt.time > time_open - late_open_dates = first_dts.index[f_late_open] - self.assertEqual(len(late_open_dates), 0) + dfd = dat.history(start=start_d, end=end_d, interval='1d', prepost=False, keepna=True) + self.assertTrue(_np.equal(dfd.index.date, _pd.to_datetime(last_dts.index).date).all()) def test_prune_post_intraday_asx(self): # Setup tkr = "BHP.AX" - interval_td = _dt.timedelta(hours=1) - time_open = _dt.time(10) - time_close = _dt.time(16, 12) - # No early closes in 2022 + # No early closes in 2023 dat = yf.Ticker(tkr, session=self.session) - # Test no afternoons (or mornings) were pruned - start_d = _dt.date(2022, 1, 1) - end_d = _dt.date(2022+1, 1, 1) + # Test no other afternoons (or mornings) were pruned + start_d = _dt.date(2023, 1, 1) + end_d = _dt.date(2023+1, 1, 1) df = dat.history(start=start_d, end=end_d, interval="1h", prepost=False, keepna=True) last_dts = _pd.Series(df.index).groupby(df.index.date).last() - f_early_close = (last_dts+interval_td).dt.time < time_close - early_close_dates = last_dts.index[f_early_close].values - self.assertEqual(len(early_close_dates), 0) - - first_dts = _pd.Series(df.index).groupby(df.index.date).first() - f_late_open = first_dts.dt.time > time_open - late_open_dates = first_dts.index[f_late_open] - self.assertEqual(len(late_open_dates), 0) + dfd = dat.history(start=start_d, end=end_d, interval='1d', prepost=False, keepna=True) + self.assertTrue(_np.equal(dfd.index.date, _pd.to_datetime(last_dts.index).date).all()) def test_weekly_2rows_fix(self): tkr = "AMZN" diff --git a/tests/ticker.py b/tests/ticker.py index 45f8eca29..393681335 100644 --- a/tests/ticker.py +++ b/tests/ticker.py @@ -17,7 +17,7 @@ import unittest import requests_cache -from typing import Union, Any +from typing import Union, Any, get_args, _GenericAlias from urllib.parse import urlparse, parse_qs, urlencode, urlunparse ticker_attributes = ( @@ -31,11 +31,10 @@ ("actions", pd.DataFrame), ("shares", pd.DataFrame), ("info", dict), - ("calendar", pd.DataFrame), + ("calendar", dict), ("recommendations", Union[pd.DataFrame, dict]), ("recommendations_summary", Union[pd.DataFrame, dict]), ("upgrades_downgrades", Union[pd.DataFrame, dict]), - ("recommendations_history", Union[pd.DataFrame, dict]), ("earnings", pd.DataFrame), ("quarterly_earnings", pd.DataFrame), ("quarterly_cashflow", pd.DataFrame), @@ -58,7 +57,12 @@ def assert_attribute_type(testClass: unittest.TestCase, instance, attribute_name try: attribute = getattr(instance, attribute_name) if attribute is not None and expected_type is not Any: - testClass.assertEqual(type(attribute), expected_type) + err_msg = f'{attribute_name} type is {type(attribute)} not {expected_type}' + if isinstance(expected_type, _GenericAlias) and expected_type.__origin__ is Union: + allowed_types = get_args(expected_type) + testClass.assertTrue(isinstance(attribute, allowed_types), err_msg) + else: + testClass.assertEqual(type(attribute), expected_type, err_msg) except Exception: testClass.assertRaises( YFNotImplementedError, lambda: getattr(instance, attribute_name) @@ -136,8 +140,8 @@ def test_goodTicker_withProxy(self): tkr = "IBM" dat = yf.Ticker(tkr, session=self.session, proxy=self.proxy) - dat._fetch_ticker_tz(timeout=5) - dat._get_ticker_tz(timeout=5) + dat._fetch_ticker_tz(proxy=None, timeout=5) + dat._get_ticker_tz(proxy=None, timeout=5) dat.history(period="1wk") for attribute_name, attribute_type in ticker_attributes: @@ -654,6 +658,24 @@ def test_cash_flow_alt_names(self): def test_bad_freq_value_raises_exception(self): self.assertRaises(ValueError, lambda: self.ticker.get_cashflow(freq="badarg")) + def test_calendar(self): + data = self.ticker.calendar + self.assertIsInstance(data, dict, "data has wrong type") + self.assertTrue(len(data) > 0, "data is empty") + self.assertIn("Earnings Date", data.keys(), "data missing expected key") + self.assertIn("Earnings Average", data.keys(), "data missing expected key") + self.assertIn("Earnings Low", data.keys(), "data missing expected key") + self.assertIn("Earnings High", data.keys(), "data missing expected key") + self.assertIn("Revenue Average", data.keys(), "data missing expected key") + self.assertIn("Revenue Low", data.keys(), "data missing expected key") + self.assertIn("Revenue High", data.keys(), "data missing expected key") + # dividend date is not available for tested ticker GOOGL + if self.ticker.ticker != "GOOGL": + self.assertIn("Dividend Date", data.keys(), "data missing expected key") + # ex-dividend date is not always available + data_cached = self.ticker.calendar + self.assertIs(data, data_cached, "data not cached") + # Below will fail because not ported to Yahoo API # def test_sustainability(self): @@ -664,6 +686,30 @@ def test_bad_freq_value_raises_exception(self): # data_cached = self.ticker.sustainability # self.assertIs(data, data_cached, "data not cached") + # def test_shares(self): + # data = self.ticker.shares + # self.assertIsInstance(data, pd.DataFrame, "data has wrong type") + # self.assertFalse(data.empty, "data is empty") + + +class TestTickerAnalysts(unittest.TestCase): + session = None + + @classmethod + def setUpClass(cls): + cls.session = session_gbl + + @classmethod + def tearDownClass(cls): + if cls.session is not None: + cls.session.close() + + def setUp(self): + self.ticker = yf.Ticker("GOOGL", session=self.session) + + def tearDown(self): + self.ticker = None + def test_recommendations(self): data = self.ticker.recommendations data_summary = self.ticker.recommendations_summary @@ -674,18 +720,16 @@ def test_recommendations(self): data_cached = self.ticker.recommendations self.assertIs(data, data_cached, "data not cached") - # def test_recommendations_summary(self): # currently alias for recommendations - # data = self.ticker.recommendations_summary - # self.assertIsInstance(data, pd.DataFrame, "data has wrong type") - # self.assertFalse(data.empty, "data is empty") + def test_recommendations_summary(self): # currently alias for recommendations + data = self.ticker.recommendations_summary + self.assertIsInstance(data, pd.DataFrame, "data has wrong type") + self.assertFalse(data.empty, "data is empty") - # data_cached = self.ticker.recommendations_summary - # self.assertIs(data, data_cached, "data not cached") + data_cached = self.ticker.recommendations_summary + self.assertIs(data, data_cached, "data not cached") - def test_recommendations_history(self): # alias for upgrades_downgrades + def test_upgrades_downgrades(self): data = self.ticker.upgrades_downgrades - data_history = self.ticker.recommendations_history - self.assertTrue(data.equals(data_history)) self.assertIsInstance(data, pd.DataFrame, "data has wrong type") self.assertFalse(data.empty, "data is empty") self.assertTrue(len(data.columns) == 4, "data has wrong number of columns") @@ -695,6 +739,8 @@ def test_recommendations_history(self): # alias for upgrades_downgrades data_cached = self.ticker.upgrades_downgrades self.assertIs(data, data_cached, "data not cached") + # Below will fail because not ported to Yahoo API + # def test_analyst_price_target(self): # data = self.ticker.analyst_price_target # self.assertIsInstance(data, pd.DataFrame, "data has wrong type") @@ -711,28 +757,6 @@ def test_recommendations_history(self): # alias for upgrades_downgrades # data_cached = self.ticker.revenue_forecasts # self.assertIs(data, data_cached, "data not cached") - def test_calendar(self): - data = self.ticker.calendar - self.assertIsInstance(data, dict, "data has wrong type") - self.assertTrue(len(data) > 0, "data is empty") - self.assertIn("Earnings Date", data.keys(), "data missing expected key") - self.assertIn("Earnings Average", data.keys(), "data missing expected key") - self.assertIn("Earnings Low", data.keys(), "data missing expected key") - self.assertIn("Earnings High", data.keys(), "data missing expected key") - self.assertIn("Revenue Average", data.keys(), "data missing expected key") - self.assertIn("Revenue Low", data.keys(), "data missing expected key") - self.assertIn("Revenue High", data.keys(), "data missing expected key") - # dividend date is not available for tested ticker GOOGL - if self.ticker.ticker != "GOOGL": - self.assertIn("Dividend Date", data.keys(), "data missing expected key") - # ex-dividend date is not always available - data_cached = self.ticker.calendar - self.assertIs(data, data_cached, "data not cached") - - # def test_shares(self): - # data = self.ticker.shares - # self.assertIsInstance(data, pd.DataFrame, "data has wrong type") - # self.assertFalse(data.empty, "data is empty") class TestTickerInfo(unittest.TestCase): @@ -777,11 +801,11 @@ def test_complementary_info(self): # We don't expect this one to have a trailing PEG ratio data1 = self.tickers[0].info - self.assertEqual(data1['trailingPegRatio'], None) + self.assertIsNone(data1['trailingPegRatio']) # This one should have a trailing PEG ratio data2 = self.tickers[2].info - self.assertEqual(data2['trailingPegRatio'], 1.2713) + self.assertIsInstance(data2['trailingPegRatio'], float) pass # def test_fast_info_matches_info(self): diff --git a/yfinance/base.py b/yfinance/base.py index 406856a67..d2f3c9277 100644 --- a/yfinance/base.py +++ b/yfinance/base.py @@ -86,7 +86,6 @@ def history(self, period="1mo", interval="1d", start=None, end=None, prepost=False, actions=True, auto_adjust=True, back_adjust=False, repair=False, keepna=False, proxy=None, rounding=False, timeout=10, - debug=None, # deprecated raise_errors=False) -> pd.DataFrame: """ :Parameters: @@ -126,23 +125,12 @@ def history(self, period="1mo", interval="1d", If not None stops waiting for a response after given number of seconds. (Can also be a fraction of a second e.g. 0.01) Default is 10 seconds. - debug: bool - If passed as False, will suppress message printing to console. - DEPRECATED, will be removed in future version raise_errors: bool If True, then raise errors as Exceptions instead of logging. """ logger = utils.get_yf_logger() proxy = proxy or self.proxy - if debug is not None: - if debug: - utils.print_once(f"yfinance: Ticker.history(debug={debug}) argument is deprecated and will be removed in future version. Do this instead: logging.getLogger('yfinance').setLevel(logging.ERROR)") - logger.setLevel(logging.ERROR) - else: - utils.print_once(f"yfinance: Ticker.history(debug={debug}) argument is deprecated and will be removed in future version. Do this instead to suppress error messages: logging.getLogger('yfinance').setLevel(logging.CRITICAL)") - logger.setLevel(logging.CRITICAL) - start_user = start end_user = end if start or period is None or period.lower() == "max": @@ -395,9 +383,6 @@ def history(self, period="1mo", interval="1d", df = df[~df.index.duplicated(keep='first')] # must do before repair - if isinstance(repair, str) and repair=='silent': - utils.log_once(logging.WARNING, "yfinance: Ticker.history(repair='silent') value is deprecated and will be removed in future version. Repair now silent by default, use logging module to increase verbosity.") - repair = True if repair: # Do this before auto/back adjust logger.debug(f'{self.ticker}: checking OHLC for repairs ...') diff --git a/yfinance/scrapers/holders.py b/yfinance/scrapers/holders.py index 90db8bae9..17a260420 100644 --- a/yfinance/scrapers/holders.py +++ b/yfinance/scrapers/holders.py @@ -1,7 +1,9 @@ # from io import StringIO import pandas as pd +import requests +from yfinance import utils from yfinance.data import YfData from yfinance.const import _BASE_URL_ from yfinance.exceptions import YFinanceDataException @@ -76,7 +78,21 @@ def _fetch(self, proxy): return result def _fetch_and_parse(self): - result = self._fetch(self.proxy) + try: + result = self._fetch(self.proxy) + except requests.exceptions.HTTPError as e: + utils.get_yf_logger().error(str(e)) + + self._major = pd.DataFrame() + self._major_direct_holders = pd.DataFrame() + self._institutional = pd.DataFrame() + self._mutualfund = pd.DataFrame() + self._insider_transactions = pd.DataFrame() + self._insider_purchases = pd.DataFrame() + self._insider_roster = pd.DataFrame() + + return + try: data = result["quoteSummary"]["result"][0] # parse "institutionOwnership", "fundOwnership", "majorDirectHolders", "majorHoldersBreakdown", "insiderTransactions", "insiderHolders", "netSharePurchaseActivity" @@ -227,4 +243,4 @@ def _parse_net_share_purchase_activity(self, data): ).convert_dtypes() self._insider_purchases = df - \ No newline at end of file + diff --git a/yfinance/scrapers/quote.py b/yfinance/scrapers/quote.py index 9d7a67b14..1daed5327 100644 --- a/yfinance/scrapers/quote.py +++ b/yfinance/scrapers/quote.py @@ -6,6 +6,7 @@ import numpy as _np import pandas as pd +import requests from yfinance import utils from yfinance.data import YfData @@ -585,28 +586,34 @@ def sustainability(self) -> pd.DataFrame: def recommendations(self) -> pd.DataFrame: if self._recommendations is None: result = self._fetch(self.proxy, modules=['recommendationTrend']) - try: - data = result["quoteSummary"]["result"][0]["recommendationTrend"]["trend"] - except (KeyError, IndexError): - raise YFinanceDataException(f"Failed to parse json response from Yahoo Finance: {result}") - self._recommendations = pd.DataFrame(data) + if result is None: + self._recommendations = pd.DataFrame() + else: + try: + data = result["quoteSummary"]["result"][0]["recommendationTrend"]["trend"] + except (KeyError, IndexError): + raise YFinanceDataException(f"Failed to parse json response from Yahoo Finance: {result}") + self._recommendations = pd.DataFrame(data) return self._recommendations @property def upgrades_downgrades(self) -> pd.DataFrame: if self._upgrades_downgrades is None: result = self._fetch(self.proxy, modules=['upgradeDowngradeHistory']) - try: - data = result["quoteSummary"]["result"][0]["upgradeDowngradeHistory"]["history"] - if len(data) == 0: - raise YFinanceDataException(f"No upgrade/downgrade history found for {self._symbol}") - df = pd.DataFrame(data) - df.rename(columns={"epochGradeDate": "GradeDate", 'firm': 'Firm', 'toGrade': 'ToGrade', 'fromGrade': 'FromGrade', 'action': 'Action'}, inplace=True) - df.set_index('GradeDate', inplace=True) - df.index = pd.to_datetime(df.index, unit='s') - self._upgrades_downgrades = df - except (KeyError, IndexError): - raise YFinanceDataException(f"Failed to parse json response from Yahoo Finance: {result}") + if result is None: + self._upgrades_downgrades = pd.DataFrame() + else: + try: + data = result["quoteSummary"]["result"][0]["upgradeDowngradeHistory"]["history"] + if len(data) == 0: + raise YFinanceDataException(f"No upgrade/downgrade history found for {self._symbol}") + df = pd.DataFrame(data) + df.rename(columns={"epochGradeDate": "GradeDate", 'firm': 'Firm', 'toGrade': 'ToGrade', 'fromGrade': 'FromGrade', 'action': 'Action'}, inplace=True) + df.set_index('GradeDate', inplace=True) + df.index = pd.to_datetime(df.index, unit='s') + self._upgrades_downgrades = df + except (KeyError, IndexError): + raise YFinanceDataException(f"Failed to parse json response from Yahoo Finance: {result}") return self._upgrades_downgrades @property @@ -627,7 +634,11 @@ def _fetch(self, proxy, modules: list): if len(modules) == 0: raise YFinanceException("No valid modules provided, see available modules using `valid_modules`") params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol} - result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict, proxy=proxy) + try: + result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict, proxy=proxy) + except requests.exceptions.HTTPError as e: + utils.get_yf_logger().error(str(e)) + return None return result def _fetch_info(self, proxy): @@ -636,6 +647,10 @@ def _fetch_info(self, proxy): self._already_fetched = True modules = ['financialData', 'quoteType', 'defaultKeyStatistics', 'assetProfile', 'summaryDetail'] result = self._fetch(proxy, modules=modules) + if result is None: + self._info = {} + return + result["quoteSummary"]["result"][0]["symbol"] = self._symbol query1_info = next( (info for info in result.get("quoteSummary", {}).get("result", []) if info["symbol"] == self._symbol), @@ -730,6 +745,10 @@ def _fetch_complementary(self, proxy): def _fetch_calendar(self): # secFilings return too old data, so not requesting it for now result = self._fetch(self.proxy, modules=['calendarEvents']) + if result is None: + self._calendar = {} + return + try: self._calendar = dict() _events = result["quoteSummary"]["result"][0]["calendarEvents"] diff --git a/yfinance/ticker.py b/yfinance/ticker.py index f205d245b..806899ace 100644 --- a/yfinance/ticker.py +++ b/yfinance/ticker.py @@ -176,10 +176,6 @@ def recommendations_summary(self): def upgrades_downgrades(self): return self.get_upgrades_downgrades() - @property - def recommendations_history(self): - return self.get_upgrades_downgrades() - @property def earnings(self) -> _pd.DataFrame: return self.get_earnings() diff --git a/yfinance/version.py b/yfinance/version.py index b76032216..25a1d1a51 100644 --- a/yfinance/version.py +++ b/yfinance/version.py @@ -1 +1 @@ -version = "0.2.34" +version = "0.2.35"