💾 Bug: Fix Save Functions 💾 (#45)

* resolved * deps * fix * temp fix * try again
suchak1 · Oct 2, 2020 · 1f3411c · 1f3411c
1 parent 5341ff1
commit 1f3411c
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 27 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -17,7 +17,7 @@ jobs:
 
       - name: Bump version and push tag
         id: bumpVersion
-        uses: anothrNick/github-tag-action@1.26.0
+        uses: anothrNick/github-tag-action@1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           WITH_V: true

diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml
@@ -12,7 +12,6 @@ env:
   RH_USERNAME: ${{ secrets.RH_USERNAME }}
   RH_PASSWORD: ${{ secrets.RH_PASSWORD }}
   RH_2FA: ${{ secrets.RH_2FA }}
-  IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }}
   STOCKTWITS: ${{ secrets.STOCKTWITS }}
   S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
   APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }}
@@ -55,6 +54,8 @@ jobs:
           flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
 
       - name: Run all unit tests
+        env:
+          IEXCLOUD: ${{ secrets.IEXCLOUD }}
         run: coverage run -m pytest -vv
 
       - name: Generate test coverage report
@@ -64,9 +65,13 @@ jobs:
         run: python scripts/update_symbols.py
 
       - name: Update dividends
+        env:
+          IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }}
         run: python scripts/update_dividends.py
 
       - name: Update splits
+        env:
+          IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }}
         run: python scripts/update_splits.py
 
       # - name: Update OHLC

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 python-dotenv == 0.14.0
 pandas == 1.1.2
 robin-stocks == 1.5.1
-boto3 == 1.15.5
+boto3 == 1.15.9
 polygon-api-client == 0.1.8
diff --git a/src/Constants.py b/src/Constants.py
@@ -4,7 +4,8 @@
 
 load_dotenv()
 # Environment
-DEV = bool(os.environ.get('DEV'))
+DEV = bool(os.environ.get('DEV')
+           and os.environ['DEV'].lower() == 'true')
 
 # File Paths
 DATA_DIR = 'data'

diff --git a/src/DataSource.py b/src/DataSource.py
@@ -67,9 +67,10 @@ def standardize_dividends(self, symbol, df):
     def save_dividends(self, **kwargs):
         # given a symbol, save its dividend history
         symbol = kwargs['symbol']
-        df = self.get_dividends(**kwargs)
-        self.writer.update_csv(
-            self.finder.get_dividends_path(symbol, self.provider), df)
+        filename = self.finder.get_dividends_path(symbol, self.provider)
+        df = self.reader.update_df(
+            filename, self.get_dividends(**kwargs), C.EX)
+        self.writer.update_csv(filename, df)
 
     def get_splits(self, symbol, timeframe='max'):
         # given a symbol, return a cached dataframe
@@ -97,9 +98,9 @@ def standardize_splits(self, symbol, df):
     def save_splits(self, **kwargs):
         # given a symbol, save its splits history
         symbol = kwargs['symbol']
-        df = self.get_splits(**kwargs)
-        self.writer.update_csv(
-            self.finder.get_splits_path(symbol, self.provider), df)
+        filename = self.finder.get_splits_path(symbol, self.provider)
+        df = self.reader.update_df(filename, self.get_splits(**kwargs), C.EX)
+        self.writer.update_csv(filename, df)
 
     # def standardize_ohlc(self, symbol, df):
     #     full_mapping = dict(
@@ -142,8 +143,15 @@ def get_social_volume(self, symbol, timeframe='max'):
     def save_social_sentiment(self, **kwargs):
         # # given a symbol, save its sentiment data
         symbol = kwargs['symbol']
-        sen_df = self.get_social_sentiment(**kwargs)
-        vol_df = self.get_social_volume(**kwargs)
+        filename = self.finder.get_sentiment_path(symbol)
+
+        sen_df = self.reader.update_df(
+            filename, self.get_social_sentiment(**kwargs), C.TIME)
+        sen_df = sen_df[{C.TIME, C.POS, C.NEG}.intersection(sen_df.columns)]
+
+        vol_df = self.reader.update_df(
+            filename, self.get_social_volume(**kwargs), C.TIME)
+        vol_df = vol_df[{C.TIME, C.VOL, C.DELTA}.intersection(vol_df.columns)]
 
         if sen_df.empty and not vol_df.empty:
             df = vol_df
@@ -153,9 +161,7 @@ def save_social_sentiment(self, **kwargs):
             df = sen_df.merge(vol_df, how="outer", on=C.TIME)
         else:
             return
-
-        self.writer.update_csv(
-            self.finder.get_sentiment_path(symbol), df)
+        self.writer.update_csv(filename, df)
 
     def standardize_sentiment(self, symbol, df):
         full_mapping = dict(

diff --git a/test/test_DataSource.py b/test/test_DataSource.py
@@ -10,7 +10,7 @@
 md = MarketData()
 iex = IEXCloud()
 poly = Polygon()
-twits = StockTwits()
+twit = StockTwits()
 
 if not os.environ.get('CI'):
     iex.token = os.environ['IEXCLOUD_SANDBOX']
@@ -20,11 +20,10 @@
     md.reader.store.bucket_name = os.environ['S3_DEV_BUCKET']
     poly.writer.store.bucket_name = os.environ['S3_DEV_BUCKET']
     poly.reader.store.bucket_name = os.environ['S3_DEV_BUCKET']
-    twits.writer.store.bucket_name = os.environ['S3_DEV_BUCKET']
-    twits.reader.store.bucket_name = os.environ['S3_DEV_BUCKET']
-
-iex.base = 'https://sandbox.iexapis.com'
-iex.base = 'https://sandbox.iexapis.com'
+    twit.writer.store.bucket_name = os.environ['S3_DEV_BUCKET']
+    twit.reader.store.bucket_name = os.environ['S3_DEV_BUCKET']
+# TODO: uncomment this when sandbox comes back online
+# iex.base = 'https://sandbox.iexapis.com'
 exp_symbols = ['AAPL', 'FB', 'DIS']
 retries = 10
 
@@ -154,7 +153,7 @@ def test_save_social_sentiment(self):
         if os.path.exists(sent_path):
             os.rename(sent_path, temp_path)
 
-        twits.save_social_sentiment(symbol=symbol, timeframe='1d')
+        twit.save_social_sentiment(symbol=symbol, timeframe='1d')
 
         assert md.reader.check_file_exists(sent_path)
         assert md.reader.store.modified_delta(sent_path).total_seconds() < 60
@@ -270,16 +269,16 @@ def test_get_splits(self):
 
 class TestStockTwits:
     def test_init(self):
-        assert type(twits).__name__ == 'StockTwits'
-        assert hasattr(twits, 'provider')
-        assert hasattr(twits, 'token')
+        assert type(twit).__name__ == 'StockTwits'
+        assert hasattr(twit, 'provider')
+        assert hasattr(twit, 'token')
 
     def test_get_social_volume(self):
-        df = twits.get_social_volume('TSLA')
+        df = twit.get_social_volume('TSLA')
         assert len(df) > 30
         assert {C.TIME, C.VOL, C.DELTA}.issubset(df.columns)
 
     def test_get_social_sentiment(self):
-        df = twits.get_social_sentiment('TSLA')
+        df = twit.get_social_sentiment('TSLA')
         assert len(df) > 30
         assert {C.TIME, C.POS, C.NEG}.issubset(df.columns)