Various changes to help dev testing

Various changes to help dev testing
- detect tz fetch returning None
- handle get_crumb() failing in USA
- stop get_crumb() breaking price fetches
- disable _fetch_complementary() because failing

yfinance/base.py

@@ -1656,7 +1656,8 @@ def _get_ticker_tz(self, proxy, timeout):
                 # info fetch is relatively slow so cache timezone
                 cache.store(self.ticker, tz)
             else:
-                tz = None
+                # tz = None
+                raise Exception(f"_fetch_ticker_tz() has returned None for '{self.ticker}', will cause problems. Investigate")
 
         self._tz = tz
         return tz

yfinance/data.py

@@ -71,7 +71,7 @@ def _get_cookie(self, proxy=None, timeout=30):
         utils.get_yf_logger().debug(f"cookie = '{utils.cookie}'")
         return utils.cookie
 
-    def _get_crumb(self, proxy=None, timeout=30):
+    def _get_crumb_basic(self, proxy=None, timeout=30):
         if utils.crumb is not None:
             return utils.crumb
 
@@ -89,7 +89,6 @@ def _get_crumb(self, proxy=None, timeout=30):
             raise Exception("Failed to fetch crumb")
 
         utils.get_yf_logger().debug(f"crumb = '{utils.crumb}'")
-        utils.crumb = utils.crumb
         return utils.crumb
 
     def _get_crumb_botunit(self, proxy=None, timeout=30):
@@ -128,9 +127,18 @@ def _get_crumb_botunit(self, proxy=None, timeout=30):
             raise Exception("Failed to fetch crumb")
 
         utils.get_yf_logger().debug(f"crumb = '{utils.crumb}'")
-        utils.crumb = utils.crumb
         return utils.crumb
 
+    def _get_crumb(self, proxy=None, timeout=30):
+        try:
+            return self._get_crumb_botunit(proxy, timeout)
+        except Exception as e:
+            if "csrfToken" in str(e):
+                # _get_crumb_botunit() fails in USA, but not sure _get_crumb() fixes fetch
+                return self._get_crumb(proxy, timeout)
+            else:
+                raise
+
     def get(self, url, user_agent_headers=None, params=None, cookies=None, proxy=None, timeout=30):
         utils.get_yf_logger().debug(f'get(): {url}')
         proxy = self._get_proxy(proxy)
@@ -143,8 +151,9 @@ def get(self, url, user_agent_headers=None, params=None, cookies=None, proxy=Non
         if params is None:
             params = {}
         if 'crumb' not in params:
-            # params['crumb'] = self._get_crumb()
-            params['crumb'] = self._get_crumb_botunit()
+            # Be careful which URLs get a crumb, because crumb breaks some fetches
+            if 'finance/quoteSummary' in url:
+                params['crumb'] = self._get_crumb()
 
         response = self._session.get(
             url=url,

yfinance/scrapers/quote.py

@@ -569,7 +569,7 @@ def __init__(self, data: TickerData, proxy=None):
     def info(self) -> dict:
         if self._info is None:
             self._fetch(self.proxy)
-            self._fetch_complementary(self.proxy)  # Failing, don't know why. Help!
+            # self._fetch_complementary(self.proxy)  # Failing, don't know why. Help!
 
         return self._info