From 1f3411c753c4586a84ed842549e38ea26c4a0734 Mon Sep 17 00:00:00 2001 From: Krish Suchak <42231639+suchak1@users.noreply.github.com> Date: Fri, 2 Oct 2020 01:57:30 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=BE=20Bug:=20Fix=20Save=20Functions=20?= =?UTF-8?q?=F0=9F=92=BE=20(#45)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * resolved * deps * fix * temp fix * try again --- .github/workflows/release.yml | 2 +- .github/workflows/sandbox.yml | 7 ++++++- requirements.txt | 2 +- src/Constants.py | 3 ++- src/DataSource.py | 28 +++++++++++++++++----------- test/test_DataSource.py | 23 +++++++++++------------ 6 files changed, 38 insertions(+), 27 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af84585c..b9f4602c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: - name: Bump version and push tag id: bumpVersion - uses: anothrNick/github-tag-action@1.26.0 + uses: anothrNick/github-tag-action@1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index b59962f7..b52cb899 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -12,7 +12,6 @@ env: RH_USERNAME: ${{ secrets.RH_USERNAME }} RH_PASSWORD: ${{ secrets.RH_PASSWORD }} RH_2FA: ${{ secrets.RH_2FA }} - IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }} STOCKTWITS: ${{ secrets.STOCKTWITS }} S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }} APCA_API_KEY_ID: ${{ secrets.APCA_API_KEY_ID }} @@ -55,6 +54,8 @@ jobs: flake8 . --count --max-complexity=10 --max-line-length=127 --statistics - name: Run all unit tests + env: + IEXCLOUD: ${{ secrets.IEXCLOUD }} run: coverage run -m pytest -vv - name: Generate test coverage report @@ -64,9 +65,13 @@ jobs: run: python scripts/update_symbols.py - name: Update dividends + env: + IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }} run: python scripts/update_dividends.py - name: Update splits + env: + IEXCLOUD: ${{ secrets.IEXCLOUD_SANDBOX }} run: python scripts/update_splits.py # - name: Update OHLC diff --git a/requirements.txt b/requirements.txt index ff272ce1..450f22d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ python-dotenv == 0.14.0 pandas == 1.1.2 robin-stocks == 1.5.1 -boto3 == 1.15.5 +boto3 == 1.15.9 polygon-api-client == 0.1.8 diff --git a/src/Constants.py b/src/Constants.py index 7133ffd7..7122ab9b 100644 --- a/src/Constants.py +++ b/src/Constants.py @@ -4,7 +4,8 @@ load_dotenv() # Environment -DEV = bool(os.environ.get('DEV')) +DEV = bool(os.environ.get('DEV') + and os.environ['DEV'].lower() == 'true') # File Paths DATA_DIR = 'data' diff --git a/src/DataSource.py b/src/DataSource.py index 8b661a6a..2dda8278 100644 --- a/src/DataSource.py +++ b/src/DataSource.py @@ -67,9 +67,10 @@ def standardize_dividends(self, symbol, df): def save_dividends(self, **kwargs): # given a symbol, save its dividend history symbol = kwargs['symbol'] - df = self.get_dividends(**kwargs) - self.writer.update_csv( - self.finder.get_dividends_path(symbol, self.provider), df) + filename = self.finder.get_dividends_path(symbol, self.provider) + df = self.reader.update_df( + filename, self.get_dividends(**kwargs), C.EX) + self.writer.update_csv(filename, df) def get_splits(self, symbol, timeframe='max'): # given a symbol, return a cached dataframe @@ -97,9 +98,9 @@ def standardize_splits(self, symbol, df): def save_splits(self, **kwargs): # given a symbol, save its splits history symbol = kwargs['symbol'] - df = self.get_splits(**kwargs) - self.writer.update_csv( - self.finder.get_splits_path(symbol, self.provider), df) + filename = self.finder.get_splits_path(symbol, self.provider) + df = self.reader.update_df(filename, self.get_splits(**kwargs), C.EX) + self.writer.update_csv(filename, df) # def standardize_ohlc(self, symbol, df): # full_mapping = dict( @@ -142,8 +143,15 @@ def get_social_volume(self, symbol, timeframe='max'): def save_social_sentiment(self, **kwargs): # # given a symbol, save its sentiment data symbol = kwargs['symbol'] - sen_df = self.get_social_sentiment(**kwargs) - vol_df = self.get_social_volume(**kwargs) + filename = self.finder.get_sentiment_path(symbol) + + sen_df = self.reader.update_df( + filename, self.get_social_sentiment(**kwargs), C.TIME) + sen_df = sen_df[{C.TIME, C.POS, C.NEG}.intersection(sen_df.columns)] + + vol_df = self.reader.update_df( + filename, self.get_social_volume(**kwargs), C.TIME) + vol_df = vol_df[{C.TIME, C.VOL, C.DELTA}.intersection(vol_df.columns)] if sen_df.empty and not vol_df.empty: df = vol_df @@ -153,9 +161,7 @@ def save_social_sentiment(self, **kwargs): df = sen_df.merge(vol_df, how="outer", on=C.TIME) else: return - - self.writer.update_csv( - self.finder.get_sentiment_path(symbol), df) + self.writer.update_csv(filename, df) def standardize_sentiment(self, symbol, df): full_mapping = dict( diff --git a/test/test_DataSource.py b/test/test_DataSource.py index e915ebcf..2bd6e89d 100644 --- a/test/test_DataSource.py +++ b/test/test_DataSource.py @@ -10,7 +10,7 @@ md = MarketData() iex = IEXCloud() poly = Polygon() -twits = StockTwits() +twit = StockTwits() if not os.environ.get('CI'): iex.token = os.environ['IEXCLOUD_SANDBOX'] @@ -20,11 +20,10 @@ md.reader.store.bucket_name = os.environ['S3_DEV_BUCKET'] poly.writer.store.bucket_name = os.environ['S3_DEV_BUCKET'] poly.reader.store.bucket_name = os.environ['S3_DEV_BUCKET'] - twits.writer.store.bucket_name = os.environ['S3_DEV_BUCKET'] - twits.reader.store.bucket_name = os.environ['S3_DEV_BUCKET'] - -iex.base = 'https://sandbox.iexapis.com' -iex.base = 'https://sandbox.iexapis.com' + twit.writer.store.bucket_name = os.environ['S3_DEV_BUCKET'] + twit.reader.store.bucket_name = os.environ['S3_DEV_BUCKET'] +# TODO: uncomment this when sandbox comes back online +# iex.base = 'https://sandbox.iexapis.com' exp_symbols = ['AAPL', 'FB', 'DIS'] retries = 10 @@ -154,7 +153,7 @@ def test_save_social_sentiment(self): if os.path.exists(sent_path): os.rename(sent_path, temp_path) - twits.save_social_sentiment(symbol=symbol, timeframe='1d') + twit.save_social_sentiment(symbol=symbol, timeframe='1d') assert md.reader.check_file_exists(sent_path) assert md.reader.store.modified_delta(sent_path).total_seconds() < 60 @@ -270,16 +269,16 @@ def test_get_splits(self): class TestStockTwits: def test_init(self): - assert type(twits).__name__ == 'StockTwits' - assert hasattr(twits, 'provider') - assert hasattr(twits, 'token') + assert type(twit).__name__ == 'StockTwits' + assert hasattr(twit, 'provider') + assert hasattr(twit, 'token') def test_get_social_volume(self): - df = twits.get_social_volume('TSLA') + df = twit.get_social_volume('TSLA') assert len(df) > 30 assert {C.TIME, C.VOL, C.DELTA}.issubset(df.columns) def test_get_social_sentiment(self): - df = twits.get_social_sentiment('TSLA') + df = twit.get_social_sentiment('TSLA') assert len(df) > 30 assert {C.TIME, C.POS, C.NEG}.issubset(df.columns)