📅 Feature: Add Intraday 📅 (#71)

* resolved

* add mergers to planned data collection in readme

* add get_intraday_path

* comments

* add institutional sentiment to readme data checklista

* to compile

* limit polygon save timeframe to prevent retroactive weirdness

* add multiprocessing to speed up data updates

* delete local data files after each file update to preserve disk space

* constants tests done

* conflict

* conflict

* polygon test_intraday

* iex test_intraday

* test_save_intraday + repo rename

* readme

* git cleanup local branches

* package name change

* remove package upload in dev pipeline

* fix bugs and add volume weighted avg price to ohlc

* start generator

* polygon intraday done?

* more progrress

* fix polygon, still need to do iex

* finish iex changes

* add intraday step to dev pipeline and fix tests

* fixed?

* last fix?

* get date from first row of intra to save properly

* more inclusive time range and lower threshold

.github/workflows/sandbox.yml

@@ -76,6 +76,9 @@ jobs:
 
       - name: Update social sentiment
         run: python scripts/update_sentiment.py
+
+      - name: Update intraday
+        run: python scripts/update_intraday.py
 
       - name: Upload repo to S3
         run: python scripts/update_repo.py

requirements.txt

@@ -1,6 +1,6 @@
 python-dotenv == 0.15.0
-pandas == 1.1.5
-robin-stocks == 1.5.3
-boto3 == 1.16.35
+pandas == 1.2.0
+robin-stocks == 1.6.3
+boto3 == 1.16.51
 polygon-api-client == 0.1.9
-pytz == 2020.4
+pytz == 2020.5

scripts/update_hist_dividends.py

@@ -19,9 +19,7 @@
 def update_iex_dividends():
     for symbol in symbols:
         filename = PathFinder().get_dividends_path(
-                symbol=symbol, provider=iex.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
+            symbol=symbol, provider=iex.provider)
         try:
             iex.save_dividends(symbol=symbol, timeframe='5y')
         except Exception as e:
@@ -36,9 +34,7 @@ def update_iex_dividends():
 def update_poly_dividends():
     for symbol in symbols:
         filename = PathFinder().get_dividends_path(
-                symbol=symbol, provider=poly.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
+            symbol=symbol, provider=poly.provider)
         try:
             poly.save_dividends(symbol=symbol, timeframe='max')
         except Exception as e:

scripts/update_hist_ohlc.py

@@ -21,8 +21,6 @@ def update_iex_ohlc():
     for symbol in stock_symbols:
         filename = PathFinder().get_ohlc_path(
             symbol=symbol, provider=iex.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
         try:
             iex.save_ohlc(symbol=symbol, timeframe='max')
         except Exception as e:
@@ -38,8 +36,6 @@ def update_poly_stocks_ohlc():
     for symbol in stock_symbols:
         filename = PathFinder().get_ohlc_path(
             symbol=symbol, provider=poly_stocks.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
         try:
             poly_stocks.save_ohlc(symbol=symbol, timeframe='max')
         except Exception as e:
@@ -52,11 +48,10 @@ def update_poly_stocks_ohlc():
 
 
 def update_poly_crypto_ohlc():
+    calls_per_min = 5
     for idx, symbol in enumerate(crypto_symbols):
         filename = PathFinder().get_ohlc_path(
             symbol=symbol, provider=poly_crypto.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
         try:
             poly_crypto.save_ohlc(symbol=symbol, timeframe='max')
         except Exception as e:
@@ -67,7 +62,7 @@ def update_poly_crypto_ohlc():
                 os.remove(filename)
 
             if idx != len(crypto_symbols) - 1:
-                sleep(60 // len(crypto_symbols) + 5)
+                sleep(60 // calls_per_min + 5)
 
 
 p1 = Process(target=update_iex_ohlc)

scripts/update_hist_splits.py

@@ -19,9 +19,7 @@
 def update_iex_splits():
     for symbol in symbols:
         filename = PathFinder().get_splits_path(
-                symbol=symbol, provider=iex.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
+            symbol=symbol, provider=iex.provider)
         try:
             iex.save_splits(symbol=symbol, timeframe='5y')
         except Exception as e:
@@ -37,8 +35,6 @@ def update_poly_splits():
     for symbol in symbols:
         filename = PathFinder().get_splits_path(
             symbol=symbol, provider=poly.provider)
-        if os.path.exists(filename):
-            os.remove(filename)
         try:
             poly.save_splits(symbol=symbol, timeframe='max')
         except Exception as e:

scripts/update_intraday.py

@@ -0,0 +1,85 @@
+import os
+import sys
+from time import sleep
+from multiprocessing import Process
+sys.path.append('src')
+from DataSource import IEXCloud, Polygon  # noqa autopep8
+from Constants import PathFinder  # noqa autopep8
+import Constants as C  # noqa autopep8
+
+iex = IEXCloud()
+poly_stocks = Polygon()
+poly_crypto = Polygon(os.environ['POLYGON'])
+stock_symbols = iex.get_symbols()
+crypto_symbols = C.POLY_CRYPTO_SYMBOLS
+yesterday = iex.traveller.dates_in_range('1d')[0]
+# Double redundancy
+
+# 1st pass
+
+
+def update_iex_intraday():
+    for symbol in stock_symbols:
+        try:
+            iex.save_intraday(symbol=symbol, timeframe='1d',
+                              retries=1 if C.TEST else C.DEFAULT_RETRIES)
+        except Exception as e:
+            print(f'IEX Cloud intraday update failed for {symbol}.')
+            print(e)
+        finally:
+            filename = PathFinder().get_intraday_path(
+                symbol=symbol,
+                date=yesterday,
+                provider=iex.provider)
+            if C.CI and os.path.exists(filename):
+                os.remove(filename)
+# 2nd pass
+
+
+def update_poly_stocks_intraday():
+    for symbol in stock_symbols:
+        try:
+            poly_stocks.save_intraday(
+                symbol=symbol, timeframe='1d',
+                retries=1 if C.TEST else C.DEFAULT_RETRIES)
+        except Exception as e:
+            print(f'Polygon.io intraday update failed for {symbol}.')
+            print(e)
+        finally:
+            filename = PathFinder().get_intraday_path(
+                symbol=symbol,
+                date=yesterday,
+                provider=poly_stocks.provider)
+            if C.CI and os.path.exists(filename):
+                os.remove(filename)
+# Crypto pass
+
+
+def update_poly_crypto_intraday():
+    calls_per_min = 5
+    for idx, symbol in enumerate(crypto_symbols):
+        try:
+            poly_crypto.save_intraday(
+                symbol=symbol, timeframe='1d',
+                retries=1 if C.TEST else C.DEFAULT_RETRIES)
+        except Exception as e:
+            print(f'Polygon.io intraday update failed for {symbol}.')
+            print(e)
+        finally:
+            filename = PathFinder().get_intraday_path(
+                symbol=symbol,
+                date=yesterday,
+                provider=poly_crypto.provider)
+            if C.CI and os.path.exists(filename):
+                os.remove(filename)
+
+            if idx != len(crypto_symbols) - 1:
+                sleep(60 // calls_per_min + 5)
+
+
+p1 = Process(target=update_iex_intraday)
+p2 = Process(target=update_poly_stocks_intraday)
+p3 = Process(target=update_poly_crypto_intraday)
+p1.start()
+p2.start()
+p3.start()

scripts/update_ohlc.py

@@ -50,6 +50,7 @@ def update_poly_stocks_ohlc():
 
 
 def update_poly_crypto_ohlc():
+    calls_per_min = 5
     for idx, symbol in enumerate(crypto_symbols):
         try:
             poly_crypto.save_ohlc(symbol=symbol, timeframe='1d',
@@ -64,7 +65,7 @@ def update_poly_crypto_ohlc():
                 os.remove(filename)
 
             if idx != len(crypto_symbols) - 1:
-                sleep(60 // len(crypto_symbols) + 5)
+                sleep(60 // calls_per_min + 5)
 
 
 p1 = Process(target=update_iex_ohlc)

src/Constants.py

@@ -1,6 +1,7 @@
 import os
 from pathlib import Path
 from dotenv import load_dotenv
+from pytz import timezone
 
 load_dotenv()
 
@@ -61,6 +62,10 @@ def get_env_bool(var_name):
 AVG = 'Avg'
 TRADES = 'Trades'
 
+# Time
+TZ = timezone('US/Eastern')
+DATE_FMT = '%Y-%m-%d'
+
 # Sentiment
 POS = 'Pos'
 NEG = 'Neg'
@@ -135,14 +140,15 @@ def get_sentiment_path(self, symbol, provider='stocktwits'):
             f'{symbol.upper()}.csv'
         )
 
-    def get_intraday_path(self, symbol, provider='iexcloud'):
+    def get_intraday_path(self, symbol, date, provider='iexcloud'):
         # given a symbol,
         # return the path to its intraday ohlc data
         return os.path.join(
             DATA_DIR,
             INTRA_DIR,
             folders[provider],
-            f'{symbol.upper()}.csv'
+            symbol.upper(),
+            f'{date}.csv'
         )
 
     def get_all_paths(self, path, truncate=False):