🎊 Feature: Get and Store Splits 🎊

.github/dependabot.yml

@@ -9,3 +9,9 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "monthly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions monthly
+      interval: "monthly"

.github/workflows/sandbox.yml

@@ -65,5 +65,8 @@ jobs:
       - name: Update dividends
         run: python scripts/update_dividends.py
 
+      - name: Update splits
+        run: python scripts/update_splits.py
+
       - name: Upload repo to S3
         run: python3 scripts/update_repo.py

.github/workflows/splits.yml

@@ -0,0 +1,47 @@
+# This workflow will automatically update data files
+# For more information see: https://help.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule
+
+name: Splits
+
+on:
+  schedule:
+    - cron: "30 12 1 * *"
+    # 8:30am EST
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Update splits
+        env:
+          IEXCLOUD: ${{ secrets.IEXCLOUD }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
+        run: python scripts/update_splits.py

README.md

@@ -70,9 +70,9 @@ Using Robinhood 2FA, we can simply provide our MFA one-time password in the `.en
   - [x] ![Symbols](https://github.com/suchak1/scarlett/workflows/Symbols/badge.svg)
   - [ ] EOD OHLCV
   - [ ] Intraday OHLCV 5 min ticks
-- [ ] Actions
+- [x] Actions
   - [x] ![Dividends](https://github.com/suchak1/scarlett/workflows/Dividends/badge.svg)
-  - [ ] Splits
+  - [x] ![Splits](https://github.com/suchak1/scarlett/workflows/Splits/badge.svg)
 - [ ] Sentiment
   - [ ] News Sentiment
   - [ ] Social Sentiment

scripts/update_splits.py

@@ -0,0 +1,24 @@
+import sys
+sys.path.append('src')
+from DataSource import IEXCloud, Polygon  # noqa autopep8
+
+iex = IEXCloud()
+poly = Polygon()
+symbols = iex.get_symbols()
+
+# Double redundancy
+
+for symbol in symbols:
+    # 1st pass
+    try:
+        iex.save_splits(symbol=symbol, timeframe='3m')
+    except Exception as e:
+        print(f'IEX Cloud split update failed for {symbol}.')
+        print(e)
+
+    # 2nd pass
+    try:
+        poly.save_splits(symbol=symbol, timeframe='max')
+    except Exception as e:
+        print(f'Polygon.io split update failed for {symbol}.')
+        print(e)

src/Constants.py

@@ -32,6 +32,9 @@
 DEC = 'Dec'  # Declaration Date
 PAY = 'Pay'  # Payment Date
 
+# Splits
+RATIO = 'Ratio'
+
 
 class PathFinder:
     def make_path(self, path):
@@ -54,12 +57,13 @@ def get_dividends_path(self, symbol, provider='iexcloud'):
             f'{symbol.upper()}.csv'
         )
 
-    def get_splits_path(self, symbol):
+    def get_splits_path(self, symbol, provider='iexcloud'):
         # given a symbol
         # return the path to its stock splits
         return os.path.join(
             DATA_DIR,
             SPLT_DIR,
+            folders[provider],
             f'{symbol.upper()}.csv'
         )
 

src/DataSource.py

@@ -30,25 +30,36 @@ def get_dividends(self, symbol, timeframe='max'):
         filtered = self.reader.data_in_timeframe(df, C.EX, timeframe)
         return filtered
 
+    def standardize(self, symbol, df, full_mapping, fx, columns, default):
+        mapping = {k: v for k, v in full_mapping.items() if k in df}
+
+        df = df[list(mapping)].rename(columns=mapping)
+        filename = fx(symbol, self.provider)
+        time_col, val_col = columns
+
+        if time_col in df and val_col in df:
+            df = self.reader.update_df(
+                filename, df, time_col).sort_values(by=[time_col])
+            df[val_col] = df[val_col].apply(
+                lambda val: float(val) if val else default)
+
+        return df
+
     def standardize_dividends(self, symbol, df):
         full_mapping = dict(
             zip(
                 ['exDate', 'paymentDate', 'declaredDate', 'amount'],
                 [C.EX, C.PAY, C.DEC, C.DIV]
             )
         )
-        mapping = {k: v for k, v in full_mapping.items() if k in df}
-        columns = list(mapping)
-
-        df = df[columns].rename(columns=mapping)
-        filename = self.finder.get_dividends_path(symbol, self.provider)
-
-        if C.EX in df and C.DIV in df:
-            df = self.reader.update_df(
-                filename, df, C.EX).sort_values(by=[C.EX])
-            df[C.DIV] = df[C.DIV].apply(lambda amt: float(amt) if amt else 0)
-
-        return df
+        return self.standardize(
+            symbol,
+            df,
+            full_mapping,
+            self.finder.get_dividends_path,
+            [C.EX, C.DIV],
+            0
+        )
 
     def save_dividends(self, **kwargs):
         # given a symbol, save its dividend history
@@ -57,18 +68,35 @@ def save_dividends(self, **kwargs):
         self.writer.update_csv(
             self.finder.get_dividends_path(symbol, self.provider), df)
 
-    # def get_splits(self, symbol, timeframe='max'):
-    #     # given a symbol, return a cached dataframe
-    #     df = self.reader.load_csv(self.finder.get_splits_path(symbol))
-    #     filtered = self.reader.data_in_timeframe(df, C.EX, timeframe)
-    #     return filtered
+    def get_splits(self, symbol, timeframe='max'):
+        # given a symbol, return a cached dataframe
+        df = self.reader.load_csv(
+            self.finder.get_splits_path(symbol, self.provider))
+        filtered = self.reader.data_in_timeframe(df, C.EX, timeframe)
+        return filtered
 
-    # def save_splits(self, **kwargs):
-    #     # given a symbol, save its splits history
-    #     symbol = kwargs['symbol']
-    #     df = self.get_splits(**kwargs)
-    #     self.writer.update_csv(self.finder.get_splits_path(symbol), df)
+    def standardize_splits(self, symbol, df):
+        full_mapping = dict(
+            zip(
+                ['exDate', 'paymentDate', 'declaredDate', 'ratio'],
+                [C.EX, C.PAY, C.DEC, C.RATIO]
+            )
+        )
+        return self.standardize(
+            symbol,
+            df,
+            full_mapping,
+            self.finder.get_splits_path,
+            [C.EX, C.RATIO],
+            1
+        )
 
+    def save_splits(self, **kwargs):
+        # given a symbol, save its splits history
+        symbol = kwargs['symbol']
+        df = self.get_splits(**kwargs)
+        self.writer.update_csv(
+            self.finder.get_splits_path(symbol, self.provider), df)
 
 # make tiingo OR IEX CLOUD!! version of get dividends which
 # fetches existing dividend csv and adds a row if dividend
@@ -121,15 +149,34 @@ def get_dividends(self, symbol, timeframe='3m'):
 
         return self.standardize_dividends(symbol, df)
 
-    # def get_splits(self, symbol):
-    #     # given a symbol, return the stock splits
-    #     ticker = yf.Ticker(symbol.replace('.', '-'))
-    #     df = ticker.actions.reset_index().drop(
-    #         'Dividends',
-    #         axis=1
-    #     )
-    #     df = df[df['Stock Splits'] != 0]
-    #     return df
+    def get_splits(self, symbol, timeframe='3m'):
+        # given a symbol, return the stock splits
+        category = 'stock'
+        dataset = 'splits'
+        parts = [
+            self.base,
+            self.version,
+            category,
+            symbol.lower(),
+            dataset,
+            timeframe
+        ]
+        endpoint = self.get_endpoint(parts)
+        response = requests.get(endpoint)
+        empty = pd.DataFrame()
+
+        if response.ok:
+            data = response.json()
+            # self.writer.save_json(f'data/{symbol}.json', data)
+        else:
+            print(f'Invalid response from IEX for {symbol} splits.')
+
+        if not response.ok or data == []:
+            return empty
+
+        df = pd.DataFrame(data)
+
+        return self.standardize_splits(symbol, df)
 
 
 class Polygon(MarketData):
@@ -144,3 +191,11 @@ def get_dividends(self, symbol, timeframe='max'):
         raw = pd.DataFrame(response.results)
         df = self.standardize_dividends(symbol, raw)
         return self.reader.data_in_timeframe(df, C.EX, timeframe)
+
+    def get_splits(self, symbol, timeframe='max'):
+        response = self.client.reference_stock_splits(symbol)
+        raw = pd.DataFrame(response.results)
+        df = self.standardize_splits(symbol, raw)
+        return self.reader.data_in_timeframe(df, C.EX, timeframe)
+
+# newShares = oldShares / ratio

test/test_Constants.py

@@ -20,5 +20,7 @@ def test_get_dividends_path(self):
             'AMD') == 'data/dividends/iexcloud/AMD.csv'
 
     def test_get_splits_path(self):
-        assert finder.get_splits_path('aapl') == 'data/splits/AAPL.csv'
-        assert finder.get_splits_path('AMD') == 'data/splits/AMD.csv'
+        assert finder.get_splits_path(
+            'aapl') == 'data/splits/iexcloud/AAPL.csv'
+        assert finder.get_splits_path(
+            'AMD') == 'data/splits/iexcloud/AMD.csv'