🆓 feat: handle polygon api changes 🆓 (#92)

* changes

* upgrade deps

* fix

* add print statements if tests are skipped during update

.github/workflows/build.yml

@@ -15,7 +15,7 @@ env:
   IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }}
   STOCKTWITS: ${{ secrets.STOCKTWITS }}
   BLS: ${{ secrets.BLS }}
-  APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+  POLYGON: ${{ secrets.POLYGON }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}

.github/workflows/dividends.yml

@@ -43,5 +43,5 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+          POLYGON: ${{ secrets.POLYGON }}
         run: python scripts/update_dividends.py

.github/workflows/intraday.yml

@@ -5,8 +5,8 @@ name: Intraday
 
 on:
   schedule:
-    - cron: "0 16 * * *"
-    # 12pm EST every day (for crypto)
+    - cron: "30 19 * * *"
+    # 2:30pm EST every day (for crypto)
 
 jobs:
   build:
@@ -43,6 +43,5 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
           POLYGON: ${{ secrets.POLYGON }}
         run: python scripts/update_intraday.py

.github/workflows/ohlc.yml

@@ -5,7 +5,7 @@ name: OHLC
 
 on:
   schedule:
-    - cron: "0 15 * * *"
+    - cron: "0 16 * * *"
     # 11am EST every day (for crypto)
 
 jobs:
@@ -43,6 +43,5 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
           POLYGON: ${{ secrets.POLYGON }}
         run: python scripts/update_ohlc.py

.github/workflows/sandbox.yml

@@ -15,7 +15,6 @@ env:
   IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }}
   STOCKTWITS: ${{ secrets.STOCKTWITS }}
   S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
-  APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
   POLYGON: ${{ secrets.POLYGON }}
   BLS: ${{ secrets.BLS }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -66,20 +65,20 @@ jobs:
       - name: Update symbols
         run: python scripts/update_symbols.py
 
-      - name: Update dividends
-        run: python scripts/update_dividends.py
+      # - name: Update dividends
+      #   run: python scripts/update_dividends.py
 
-      - name: Update splits
-        run: python scripts/update_splits.py
+      # - name: Update splits
+      #   run: python scripts/update_splits.py
 
-      - name: Update OHLC
-        run: python scripts/update_ohlc.py
+      # - name: Update OHLC
+      #   run: python scripts/update_ohlc.py
 
       - name: Update social sentiment
         run: python scripts/update_sentiment.py
 
-      - name: Update intraday
-        run: python scripts/update_intraday.py
+      # - name: Update intraday
+      #   run: python scripts/update_intraday.py
 
       - name: Update unemployment
         run: python scripts/update_unrate.py

.github/workflows/splits.yml

@@ -5,8 +5,8 @@ name: Splits
 
 on:
   schedule:
-    - cron: "30 12 1 * *"
-    # 8:30am EST
+    - cron: "30 13 1 * *"
+    # 9:30am EST
 
 jobs:
   build:
@@ -43,5 +43,5 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           S3_BUCKET: ${{ secrets.S3_BUCKET }}
-          APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+          POLYGON: ${{ secrets.POLYGON }}
         run: python scripts/update_splits.py

.github/workflows/unemployment.yml

@@ -1,7 +1,7 @@
 # This workflow will automatically update data files
 # For more information see: https://help.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule
 
-name: Unemployment Rate
+name: Unemployment
 
 on:
   schedule:

README.md

@@ -84,7 +84,7 @@ Using Robinhood 2FA, we can simply provide our MFA one-time password in the `.en
   - [ ] Cash Flow
   - [ ] CEO Compensation
 - [ ] Government / Macro           <!-- this stuff prob won't be v useful -->  
-  - [x] [![Unemployment Rate](<https://github.com/suchak1/hyperdrive/workflows/Unemployment%20Rate/badge.svg>)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3A%22Unemployment+Rate%22)          <!-- BLS -->
+  - [x] [![Unemployment](<https://github.com/suchak1/hyperdrive/workflows/Unemployment/badge.svg>)](https://github.com/suchak1/hyperdrive/actions?query=workflow%3AUnemployment)          <!-- BLS -->
   - [ ] Real GDP                   <!-- BEA -->
   - [ ] US Recession Probabilities <!-- FRED -->
 - [ ] Market

requirements.txt

@@ -1,6 +1,6 @@
 python-dotenv == 0.15.0
-pandas == 1.2.1
-robin-stocks == 1.7.0
-boto3 == 1.16.63
+pandas == 1.2.2
+robin-stocks == 2.0.0
+boto3 == 1.17.12
 polygon-api-client == 0.1.9
-pytz == 2020.5
+pytz == 2021.1

scripts/update_hist_intraday.py

@@ -2,23 +2,19 @@
 import sys
 from time import sleep
 from datetime import datetime
-from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS, POLY_CRYPTO_DELAY  # noqa autopep8
+from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS  # noqa autopep8
 
 
-poly_stocks = Polygon()
-poly_crypto = Polygon(os.environ['POLYGON'])
-stock_symbols = poly_stocks.get_symbols()
-# last_completed_stock = 'FOXA'
-# stock_symbols = stock_symbols[stock_symbols.index(last_completed_stock)+1:]
-# [250:]
+poly = Polygon(os.environ['POLYGON'])
+stock_symbols = poly.get_symbols()
 crypto_symbols = POLY_CRYPTO_SYMBOLS
+all_symbols = stock_symbols + crypto_symbols
 
 
-def update_poly_stocks_intraday():
-    for symbol in stock_symbols:
+def update_poly_intraday():
+    for symbol in all_symbols:
         now = datetime.now()
         hour = now.hour
         while hour in set(range(8, 12)):
@@ -28,7 +24,7 @@ def update_poly_stocks_intraday():
             hour = datetime.now().hour
         filenames = []
         try:
-            filenames = poly_stocks.save_intraday(
+            filenames = poly.save_intraday(
                 symbol=symbol, timeframe='30d', retries=1)
         except Exception as e:
             print(f'Polygon.io intraday update failed for {symbol}.')
@@ -38,32 +34,6 @@ def update_poly_stocks_intraday():
                 for filename in filenames:
                     if os.path.exists(filename):
                         os.remove(filename)
-# Crypto pass
 
 
-def update_poly_crypto_intraday():
-
-    for idx, symbol in enumerate(crypto_symbols):
-        filenames = []
-        try:
-            filenames = poly_crypto.save_intraday(
-                symbol=symbol, timeframe='9m',
-                delay=POLY_CRYPTO_DELAY, retries=2)
-        except Exception as e:
-            print(f'Polygon.io intraday update failed for {symbol}.')
-            print(e)
-        finally:
-            if CI:
-                for filename in filenames:
-                    if os.path.exists(filename):
-                        os.remove(filename)
-
-            if idx != len(crypto_symbols) - 1:
-                sleep(POLY_CRYPTO_DELAY)
-
-
-p2 = Process(target=update_poly_stocks_intraday)
-# p3 = Process(target=update_poly_crypto_intraday)
-if __name__ == '__main__':
-    p2.start()
-# p3.start()
+update_poly_intraday()

scripts/update_hist_ohlc.py

@@ -1,18 +1,16 @@
 import os
 import sys
-from time import sleep
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS, POLY_CRYPTO_DELAY  # noqa autopep8
+from Constants import CI, PathFinder, POLY_CRYPTO_SYMBOLS  # noqa autopep8
 
 
 iex = IEXCloud()
-poly_stocks = Polygon()
-poly_crypto = Polygon(os.environ['POLYGON'])
+poly = Polygon(os.environ['POLYGON'])
 stock_symbols = iex.get_symbols()
-# [250:]
 crypto_symbols = POLY_CRYPTO_SYMBOLS
+all_symbols = stock_symbols + crypto_symbols
 timeframe = '2m'
 # Double redundancy
 
@@ -34,41 +32,21 @@ def update_iex_ohlc():
 # 2nd pass
 
 
-def update_poly_stocks_ohlc():
-    for symbol in stock_symbols:
-        filename = PathFinder().get_ohlc_path(
-            symbol=symbol, provider=poly_stocks.provider)
-        try:
-            poly_stocks.save_ohlc(symbol=symbol, timeframe=timeframe)
-        except Exception as e:
-            print(f'Polygon.io OHLC update failed for {symbol}.')
-            print(e)
-        finally:
-            if CI and os.path.exists(filename):
-                os.remove(filename)
-# Crypto pass
-
-
-def update_poly_crypto_ohlc():
-    for idx, symbol in enumerate(crypto_symbols):
+def update_poly_ohlc():
+    for symbol in all_symbols:
         filename = PathFinder().get_ohlc_path(
-            symbol=symbol, provider=poly_crypto.provider)
+            symbol=symbol, provider=poly.provider)
         try:
-            poly_crypto.save_ohlc(symbol=symbol, timeframe=timeframe)
+            poly.save_ohlc(symbol=symbol, timeframe=timeframe)
         except Exception as e:
             print(f'Polygon.io OHLC update failed for {symbol}.')
             print(e)
         finally:
             if CI and os.path.exists(filename):
                 os.remove(filename)
 
-            if idx != len(crypto_symbols) - 1:
-                sleep(POLY_CRYPTO_DELAY)
-
 
 p1 = Process(target=update_iex_ohlc)
-p2 = Process(target=update_poly_stocks_ohlc)
-p3 = Process(target=update_poly_crypto_ohlc)
+p2 = Process(target=update_poly_ohlc)
 p1.start()
 p2.start()
-p3.start()

scripts/update_intraday.py

@@ -1,18 +1,16 @@
 import os
 import sys
-from time import sleep
 from multiprocessing import Process
 sys.path.append('src')
 from DataSource import IEXCloud, Polygon  # noqa autopep8
-from Constants import PathFinder, POLY_CRYPTO_DELAY  # noqa autopep8
+from Constants import PathFinder, POLY_CRYPTO_SYMBOLS, FEW_DAYS  # noqa autopep8
 import Constants as C  # noqa autopep8
 
 iex = IEXCloud()
-poly_stocks = Polygon()
-poly_crypto = Polygon(os.environ['POLYGON'])
+poly = Polygon(os.environ['POLYGON'])
 stock_symbols = iex.get_symbols()
-crypto_symbols = C.POLY_CRYPTO_SYMBOLS
-few_days = '3d'
+crypto_symbols = POLY_CRYPTO_SYMBOLS
+all_symbols = stock_symbols + crypto_symbols
 
 # Double redundancy
 
@@ -37,31 +35,12 @@ def update_iex_intraday():
 # 2nd pass
 
 
-def update_poly_stocks_intraday():
-    for symbol in stock_symbols:
-        filenames = []
-        try:
-            filenames = poly_stocks.save_intraday(
-                symbol=symbol, timeframe=few_days,
-                retries=1 if C.TEST else C.DEFAULT_RETRIES)
-        except Exception as e:
-            print(f'Polygon.io intraday update failed for {symbol}.')
-            print(e)
-        finally:
-            if C.CI:
-                for filename in filenames:
-                    if os.path.exists(filename):
-                        os.remove(filename)
-# Crypto pass
-
-
-def update_poly_crypto_intraday():
-    for idx, symbol in enumerate(crypto_symbols):
+def update_poly_intraday():
+    for symbol in all_symbols:
         filenames = []
         try:
-            filenames = poly_crypto.save_intraday(
-                symbol=symbol, timeframe=few_days,
-                retries=1 if C.TEST else C.DEFAULT_RETRIES)
+            filenames = poly.save_intraday(
+                symbol=symbol, timeframe=FEW_DAYS, retries=1)
         except Exception as e:
             print(f'Polygon.io intraday update failed for {symbol}.')
             print(e)
@@ -71,13 +50,8 @@ def update_poly_crypto_intraday():
                     if os.path.exists(filename):
                         os.remove(filename)
 
-            if idx != len(crypto_symbols) - 1:
-                sleep(POLY_CRYPTO_DELAY)
-
 
 p1 = Process(target=update_iex_intraday)
-p2 = Process(target=update_poly_stocks_intraday)
-p3 = Process(target=update_poly_crypto_intraday)
+p2 = Process(target=update_poly_intraday)
 p1.start()
 p2.start()
-p3.start()