🧹chore: merge conflicts

suchak1 · Jan 13, 2021 · a196091 · a196091
2 parents 38869b8 + ccdba52
commit a196091
Show file tree

Hide file tree

Showing 6 changed files with 137 additions and 48 deletions.
diff --git a/scripts/update_hist_intraday.py b/scripts/update_hist_intraday.py
@@ -0,0 +1,59 @@
+import os
+import sys
+from time import sleep
+from multiprocessing import Process
+sys.path.append('src')
+from DataSource import IEXCloud, Polygon  # noqa autopep8
+from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS, POLY_CRYPTO_DELAY  # noqa autopep8
+
+
+poly_stocks = Polygon()
+poly_crypto = Polygon(os.environ['POLYGON'])
+stock_symbols = poly_stocks.get_symbols()
+stock_symbols = stock_symbols[stock_symbols.index('DIS')+1:]
+# [250:]
+crypto_symbols = [POLY_CRYPTO_SYMBOLS[0]]
+
+
+def update_poly_stocks_intraday():
+    for symbol in stock_symbols:
+        filenames = []
+        try:
+            filenames = poly_stocks.save_intraday(
+                symbol=symbol, timeframe='21y')
+        except Exception as e:
+            print(f'Polygon.io intraday update failed for {symbol}.')
+            print(e)
+        finally:
+            if CI:
+                for filename in filenames:
+                    if os.path.exists(filename):
+                        os.remove(filename)
+# Crypto pass
+
+
+def update_poly_crypto_intraday():
+
+    for idx, symbol in enumerate(crypto_symbols):
+        filenames = []
+        try:
+            filenames = poly_crypto.save_intraday(
+                symbol=symbol, timeframe='674d',
+                delay=POLY_CRYPTO_DELAY, retries=2)
+        except Exception as e:
+            print(f'Polygon.io intraday update failed for {symbol}.')
+            print(e)
+        finally:
+            if CI:
+                for filename in filenames:
+                    if os.path.exists(filename):
+                        os.remove(filename)
+
+            if idx != len(crypto_symbols) - 1:
+                sleep(POLY_CRYPTO_DELAY)
+
+
+# p2 = Process(target=update_poly_stocks_intraday)
+p3 = Process(target=update_poly_crypto_intraday)
+# p2.start()
+p3.start()
diff --git a/scripts/update_hist_ohlc.py b/scripts/update_hist_ohlc.py
@@ -4,14 +4,16 @@
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS  # noqa autopep8
+from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS, POLY_CRYPTO_DELAY  # noqa autopep8
 
 
 iex = IEXCloud()
 poly_stocks = Polygon()
 poly_crypto = Polygon(os.environ['POLYGON'])
-stock_symbols = iex.get_symbols()[250:]
+stock_symbols = iex.get_symbols()
+# [250:]
 crypto_symbols = POLY_CRYPTO_SYMBOLS
+timeframe = '2m'
 # Double redundancy
 
 # 1st pass
@@ -22,7 +24,7 @@ def update_iex_ohlc():
         filename = PathFinder().get_ohlc_path(
             symbol=symbol, provider=iex.provider)
         try:
-            iex.save_ohlc(symbol=symbol, timeframe='max')
+            iex.save_ohlc(symbol=symbol, timeframe=timeframe)
         except Exception as e:
             print(f'IEX Cloud OHLC update failed for {symbol}.')
             print(e)
@@ -37,7 +39,7 @@ def update_poly_stocks_ohlc():
         filename = PathFinder().get_ohlc_path(
             symbol=symbol, provider=poly_stocks.provider)
         try:
-            poly_stocks.save_ohlc(symbol=symbol, timeframe='max')
+            poly_stocks.save_ohlc(symbol=symbol, timeframe=timeframe)
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
             print(e)
@@ -48,12 +50,11 @@ def update_poly_stocks_ohlc():
 
 
 def update_poly_crypto_ohlc():
-    calls_per_min = 5
     for idx, symbol in enumerate(crypto_symbols):
         filename = PathFinder().get_ohlc_path(
             symbol=symbol, provider=poly_crypto.provider)
         try:
-            poly_crypto.save_ohlc(symbol=symbol, timeframe='max')
+            poly_crypto.save_ohlc(symbol=symbol, timeframe=timeframe)
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
             print(e)
@@ -62,7 +63,7 @@ def update_poly_crypto_ohlc():
                 os.remove(filename)
 
             if idx != len(crypto_symbols) - 1:
-                sleep(60 // calls_per_min + 5)
+                sleep(POLY_CRYPTO_DELAY)
 
 
 p1 = Process(target=update_iex_ohlc)

diff --git a/scripts/update_intraday.py b/scripts/update_intraday.py
@@ -4,15 +4,17 @@
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import PathFinder  # noqa autopep8
+from Constants import PathFinder, POLY_CRYPTO_DELAY  # noqa autopep8
 import Constants as C  # noqa autopep8
 
 iex = IEXCloud()
 poly_stocks = Polygon()
 poly_crypto = Polygon(os.environ['POLYGON'])
 stock_symbols = iex.get_symbols()
 crypto_symbols = C.POLY_CRYPTO_SYMBOLS
-yesterday = iex.traveller.dates_in_range('1d')[0]
+few_days = '3d'
+last_few_days = iex.traveller.dates_in_range(few_days)
+
 # Double redundancy
 
 # 1st pass
@@ -29,7 +31,7 @@ def update_iex_intraday():
         finally:
             filename = PathFinder().get_intraday_path(
                 symbol=symbol,
-                date=yesterday,
+                date=last_few_days[-1],
                 provider=iex.provider)
             if C.CI and os.path.exists(filename):
                 os.remove(filename)
@@ -40,41 +42,43 @@ def update_poly_stocks_intraday():
     for symbol in stock_symbols:
         try:
             poly_stocks.save_intraday(
-                symbol=symbol, timeframe='1d',
+                symbol=symbol, timeframe=few_days,
                 retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io intraday update failed for {symbol}.')
             print(e)
         finally:
-            filename = PathFinder().get_intraday_path(
-                symbol=symbol,
-                date=yesterday,
-                provider=poly_stocks.provider)
-            if C.CI and os.path.exists(filename):
-                os.remove(filename)
+            for date in few_days:
+                filename = PathFinder().get_intraday_path(
+                    symbol=symbol,
+                    date=date,
+                    provider=poly_stocks.provider)
+                if C.CI and os.path.exists(filename):
+                    os.remove(filename)
 # Crypto pass
 
 
 def update_poly_crypto_intraday():
-    calls_per_min = 5
+
     for idx, symbol in enumerate(crypto_symbols):
         try:
             poly_crypto.save_intraday(
-                symbol=symbol, timeframe='3d',
+                symbol=symbol, timeframe=few_days,
                 retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io intraday update failed for {symbol}.')
             print(e)
         finally:
-            filename = PathFinder().get_intraday_path(
-                symbol=symbol,
-                date=yesterday,
-                provider=poly_crypto.provider)
-            if C.CI and os.path.exists(filename):
-                os.remove(filename)
+            for date in few_days:
+                filename = PathFinder().get_intraday_path(
+                    symbol=symbol,
+                    date=date,
+                    provider=poly_crypto.provider)
+                if C.CI and os.path.exists(filename):
+                    os.remove(filename)
 
             if idx != len(crypto_symbols) - 1:
-                sleep(60 // calls_per_min + 5)
+                sleep(POLY_CRYPTO_DELAY)
 
 
 p1 = Process(target=update_iex_intraday)

diff --git a/scripts/update_ohlc.py b/scripts/update_ohlc.py
@@ -4,14 +4,16 @@
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import PathFinder  # noqa autopep8
+from Constants import PathFinder, POLY_CRYPTO_DELAY  # noqa autopep8
 import Constants as C  # noqa autopep8
 
 iex = IEXCloud()
 poly_stocks = Polygon()
 poly_crypto = Polygon(os.environ['POLYGON'])
 stock_symbols = iex.get_symbols()
 crypto_symbols = C.POLY_CRYPTO_SYMBOLS
+few_days = '3d'
+
 # Double redundancy
 
 # 1st pass
@@ -36,7 +38,7 @@ def update_iex_ohlc():
 def update_poly_stocks_ohlc():
     for symbol in stock_symbols:
         try:
-            poly_stocks.save_ohlc(symbol=symbol, timeframe='1d',
+            poly_stocks.save_ohlc(symbol=symbol, timeframe=few_days,
                                   retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
@@ -50,10 +52,10 @@ def update_poly_stocks_ohlc():
 
 
 def update_poly_crypto_ohlc():
-    calls_per_min = 5
+
     for idx, symbol in enumerate(crypto_symbols):
         try:
-            poly_crypto.save_ohlc(symbol=symbol, timeframe='1d',
+            poly_crypto.save_ohlc(symbol=symbol, timeframe=few_days,
                                   retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
@@ -65,7 +67,7 @@ def update_poly_crypto_ohlc():
                 os.remove(filename)
 
             if idx != len(crypto_symbols) - 1:
-                sleep(60 // calls_per_min + 5)
+                sleep(POLY_CRYPTO_DELAY)
 
 
 p1 = Process(target=update_iex_ohlc)

diff --git a/src/Constants.py b/src/Constants.py
@@ -88,6 +88,7 @@ def get_env_bool(var_name):
 
 DEFAULT_RETRIES = 3
 DEFAULT_DELAY = 2
+POLY_CRYPTO_DELAY = 15
 
 
 class PathFinder: