💱 Feature: Add Crypto OHLC + 3 Retries 💱 (#52)

* making progress * try again function * iex div and splits retry * poly div and splits retry * iex ohlc retry * tests updated * fix try again * bugs * fix tests * fix retries for tests * add polygon key * timezone fix
suchak1 · Oct 28, 2020 · 4ae86d8 · 4ae86d8
1 parent ac12676
commit 4ae86d8
Show file tree

Hide file tree

Showing 13 changed files with 410 additions and 282 deletions.
diff --git a/.github/workflows/intraday.yml b/.github/workflows/intraday.yml
@@ -44,4 +44,5 @@ jobs:
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
           APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+          POLYGON: ${{ secrets.POLYGON }}
         run: python scripts/update_intraday.py
diff --git a/.github/workflows/ohlc.yml b/.github/workflows/ohlc.yml
@@ -44,4 +44,5 @@ jobs:
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
           APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+          POLYGON: ${{ secrets.POLYGON }}
         run: python scripts/update_ohlc.py
diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
@@ -16,9 +16,11 @@ env:
   STOCKTWITS: ${{ secrets.STOCKTWITS }}
   S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
   APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+  POLYGON: ${{ secrets.POLYGON }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+  TEST: true
 
 jobs:
   build:
@@ -72,5 +74,8 @@ jobs:
       - name: Update OHLC
         run: python scripts/update_ohlc.py
 
+      - name: Update social sentiment
+        run: python scripts/update_sentiment.py
+
       - name: Upload repo to S3
         run: python scripts/update_repo.py
diff --git a/requirements.txt b/requirements.txt
@@ -3,3 +3,4 @@ pandas == 1.1.2
 robin-stocks == 1.5.2
 boto3 == 1.15.9
 polygon-api-client == 0.1.8
+pytz == 2020.1
diff --git a/scripts/update_dividends.py b/scripts/update_dividends.py
@@ -3,8 +3,8 @@
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder  # noqa autopep8
-
+from Constants import PathFinder  # noqa autopep8
+import Constants as C  # noqa autopep8
 
 iex = IEXCloud()
 poly = Polygon()
@@ -18,29 +18,31 @@
 def update_iex_dividends():
     for symbol in symbols:
         try:
-            iex.save_dividends(symbol=symbol, timeframe='3m')
+            iex.save_dividends(symbol=symbol, timeframe='3m',
+                               retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'IEX Cloud dividend update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_dividends_path(
                 symbol=symbol, provider=iex.provider)
-            if CI and os.path.exists(filename):
+            if C.CI and os.path.exists(filename):
                 os.remove(filename)
 # 2nd pass
 
 
 def update_poly_dividends():
     for symbol in symbols:
         try:
-            poly.save_dividends(symbol=symbol, timeframe='3m')
+            poly.save_dividends(symbol=symbol, timeframe='3m',
+                                retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io dividend update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_dividends_path(
                 symbol=symbol, provider=poly.provider)
-            if CI and os.path.exists(filename):
+            if C.CI and os.path.exists(filename):
                 os.remove(filename)
 
 

diff --git a/scripts/update_hist_ohlc.py b/scripts/update_hist_ohlc.py
@@ -1,23 +1,24 @@
 import os
 import sys
+from time import sleep
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder  # noqa autopep8
+from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS  # noqa autopep8
 
 
 iex = IEXCloud()
-poly = Polygon()
-symbols = iex.get_symbols()
-symbols = symbols[140:]
-
+poly_stocks = Polygon()
+poly_crypto = Polygon(os.environ['POLYGON'])
+stock_symbols = iex.get_symbols()[140:]
+crypto_symbols = POLY_CRYPTO_SYMBOLS
 # Double redundancy
 
 # 1st pass
 
 
 def update_iex_ohlc():
-    for symbol in symbols:
+    for symbol in stock_symbols:
         try:
             iex.save_ohlc(symbol=symbol, timeframe='max')
         except Exception as e:
@@ -31,21 +32,41 @@ def update_iex_ohlc():
 # 2nd pass
 
 
-def update_poly_ohlc():
-    for symbol in symbols:
+def update_poly_stocks_ohlc():
+    for symbol in stock_symbols:
+        try:
+            poly_stocks.save_ohlc(symbol=symbol, timeframe='max')
+        except Exception as e:
+            print(f'Polygon.io OHLC update failed for {symbol}.')
+            print(e)
+        finally:
+            filename = PathFinder().get_ohlc_path(
+                symbol=symbol, provider=poly_stocks.provider)
+            if CI and os.path.exists(filename):
+                os.remove(filename)
+# Crypto pass
+
+
+def update_poly_crypto_ohlc():
+    for idx, symbol in enumerate(crypto_symbols):
         try:
-            poly.save_ohlc(symbol=symbol, timeframe='max')
+            poly_crypto.save_ohlc(symbol=symbol, timeframe='max')
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_ohlc_path(
-                symbol=symbol, provider=poly.provider)
+                symbol=symbol, provider=poly_crypto.provider)
             if CI and os.path.exists(filename):
                 os.remove(filename)
 
+            if idx != len(crypto_symbols) - 1:
+                sleep(60 // len(crypto_symbols) + 5)
+
 
 p1 = Process(target=update_iex_ohlc)
-p2 = Process(target=update_poly_ohlc)
+p2 = Process(target=update_poly_stocks_ohlc)
+p3 = Process(target=update_poly_crypto_ohlc)
 p1.start()
 p2.start()
+p3.start()
diff --git a/scripts/update_ohlc.py b/scripts/update_ohlc.py
@@ -1,50 +1,75 @@
 import os
 import sys
+from time import sleep
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder  # noqa autopep8
-
+from Constants import PathFinder  # noqa autopep8
+import Constants as C  # noqa autopep8
 
 iex = IEXCloud()
-poly = Polygon()
-symbols = iex.get_symbols()
-
+poly_stocks = Polygon()
+poly_crypto = Polygon(os.environ['POLYGON'])
+stock_symbols = iex.get_symbols()
+crypto_symbols = C.POLY_CRYPTO_SYMBOLS
 # Double redundancy
 
 # 1st pass
 
 
 def update_iex_ohlc():
-    for symbol in symbols:
+    for symbol in stock_symbols:
         try:
-            iex.save_ohlc(symbol=symbol, timeframe='1d')
+            iex.save_ohlc(symbol=symbol, timeframe='1d',
+                          retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'IEX Cloud OHLC update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_ohlc_path(
                 symbol=symbol, provider=iex.provider)
-            if CI and os.path.exists(filename):
+            if C.CI and os.path.exists(filename):
                 os.remove(filename)
 # 2nd pass
 
 
-def update_poly_ohlc():
-    for symbol in symbols:
+def update_poly_stocks_ohlc():
+    for symbol in stock_symbols:
+        try:
+            poly_stocks.save_ohlc(symbol=symbol, timeframe='1d',
+                                  retries=1 if C.TEST else C.DEFAULT_RETRIES)
+        except Exception as e:
+            print(f'Polygon.io OHLC update failed for {symbol}.')
+            print(e)
+        finally:
+            filename = PathFinder().get_ohlc_path(
+                symbol=symbol, provider=poly_stocks.provider)
+            if C.CI and os.path.exists(filename):
+                os.remove(filename)
+# Crypto pass
+
+
+def update_poly_crypto_ohlc():
+    for idx, symbol in enumerate(crypto_symbols):
         try:
-            poly.save_ohlc(symbol=symbol, timeframe='1d')
+            poly_crypto.save_ohlc(symbol=symbol, timeframe='1d',
+                                  retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_ohlc_path(
-                symbol=symbol, provider=poly.provider)
-            if CI and os.path.exists(filename):
+                symbol=symbol, provider=poly_crypto.provider)
+            if C.CI and os.path.exists(filename):
                 os.remove(filename)
 
+            if idx != len(crypto_symbols) - 1:
+                sleep(60 // len(crypto_symbols) + 5)
+
 
 p1 = Process(target=update_iex_ohlc)
-p2 = Process(target=update_poly_ohlc)
+p2 = Process(target=update_poly_stocks_ohlc)
+p3 = Process(target=update_poly_crypto_ohlc)
 p1.start()
 p2.start()
+p3.start()
diff --git a/scripts/update_sentiment.py b/scripts/update_sentiment.py
@@ -2,25 +2,33 @@
 import sys
 sys.path.append('src')
 from DataSource import StockTwits  # noqa autopep8
+from Constants import PathFinder  # noqa autopep8
 import Constants as C  # noqa autopep8
-from Constants import CI, PathFinder  # noqa autopep8
 
 
 twit = StockTwits()
 symbols = twit.get_symbols()
-symbols.extend(['BTC-X', 'ETH-X', 'LTC-X', 'XMR-X', 'IOT-X'])
+crypto_symbols = ['BTC-X', 'ETH-X', 'LTC-X', 'XMR-X', 'IOT-X']
+if C.TEST:
+    symbols = crypto_symbols
+    twit.token = ''
+else:
+    symbols.extend(crypto_symbols)
 BATCH = int(os.environ.get('BATCH')) if os.environ.get('BATCH') else 1
 # better solution is to dynamically choose 175 most outdated symbols
 
 # First batch
 for symbol in symbols[C.TWIT_RATE*(BATCH-1):C.TWIT_RATE*BATCH]:
+    if symbol in C.SENTIMENT_SYMBOLS_IGNORE:
+        continue
     try:
-        twit.save_social_sentiment(symbol=symbol, timeframe='1d')
+        twit.save_social_sentiment(symbol=symbol, timeframe='1d',
+                                   retries=1 if C.TEST else 2)
     except Exception as e:
         print(f'Stocktwits sentiment update failed for {symbol}.')
         print(e)
     finally:
         filename = PathFinder().get_sentiment_path(
             symbol=symbol, provider=twit.provider)
-        if CI and os.path.exists(filename):
+        if C.CI and os.path.exists(filename):
             os.remove(filename)
diff --git a/scripts/update_splits.py b/scripts/update_splits.py
@@ -3,8 +3,8 @@
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder  # noqa autopep8
-
+from Constants import PathFinder  # noqa autopep8
+import Constants as C  # noqa autopep8
 
 iex = IEXCloud()
 poly = Polygon()
@@ -18,29 +18,31 @@
 def update_iex_splits():
     for symbol in symbols:
         try:
-            iex.save_splits(symbol=symbol, timeframe='3m')
+            iex.save_splits(symbol=symbol, timeframe='3m',
+                            retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'IEX Cloud split update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_splits_path(
                 symbol=symbol, provider=iex.provider)
-            if CI and os.path.exists(filename):
+            if C.CI and os.path.exists(filename):
                 os.remove(filename)
 # 2nd pass
 
 
 def update_poly_splits():
     for symbol in symbols:
         try:
-            poly.save_splits(symbol=symbol, timeframe='3m')
+            poly.save_splits(symbol=symbol, timeframe='3m',
+                             retries=1 if C.TEST else C.DEFAULT_RETRIES)
         except Exception as e:
             print(f'Polygon.io split update failed for {symbol}.')
             print(e)
         finally:
             filename = PathFinder().get_splits_path(
                 symbol=symbol, provider=poly.provider)
-            if CI and os.path.exists(filename):
+            if C.CI and os.path.exists(filename):
                 os.remove(filename)
 
 

diff --git a/src/Constants.py b/src/Constants.py
@@ -3,11 +3,17 @@
 from dotenv import load_dotenv
 
 load_dotenv()
+
+
+def get_env_bool(var_name):
+    return bool(os.environ.get(var_name)
+                and os.environ[var_name].lower() == 'true')
+
+
 # Environment
-DEV = bool(os.environ.get('DEV')
-           and os.environ['DEV'].lower() == 'true')
-CI = bool(os.environ.get('CI')
-          and os.environ['CI'].lower() == 'true')
+DEV = get_env_bool('DEV')
+CI = get_env_bool('CI')
+TEST = get_env_bool('TEST')
 
 # File Paths
 # data
@@ -59,6 +65,22 @@
 DELTA = 'Delta'
 TWIT_RATE = 175
 
+# Misc
+POLY_CRYPTO_SYMBOLS = [
+    'X%3ABTCUSD', 'X%3AETHUSD',
+    'X%3ALTCUSD', 'X%3AXMRUSD', 'X%3AIOTUSD'
+]
+
+SENTIMENT_SYMBOLS_IGNORE = {
+    'SPYD', 'VWDRY', 'BPMP',
+    'FOX', 'YYY', 'SDIV',
+    'DIV', 'SHECY', 'PALL',
+    'DWDP', 'TFCF', 'SPAR',
+    'TMUSR', 'OXY+', 'BNTX^'}
+
+DEFAULT_RETRIES = 3
+DEFAULT_DELAY = 2
+
 
 class PathFinder:
     def make_path(self, path):