fix: Rollback Redis-py to Redis-py-cluster (feast-dev#2347)

* Roll back redis config and add tests back Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * lint Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix autogenerated import Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix lint Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Add redis-py-cluster back Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix dask ci Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Regenerate using respective python 3 versions Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Change test_entity_ttl test Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix lint Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Change integration workflow Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Temporarily disable for testing Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Continue making work Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Continue making work Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix workflow Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix workflow Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Update port numbers Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Update pr integration tests Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Reenable rest of tests Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Update data source creator Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Fix integration test Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Clean up code Signed-off-by: Kevin Zhang <kzhang@tecton.ai> * Somehow removed the key components -,- Signed-off-by: Kevin Zhang <kzhang@tecton.ai>
achals · Mar 3, 2022 · 1ba86fb · 1ba86fb
1 parent c2a6c6c
commit 1ba86fb
Show file tree

Hide file tree

Showing 10 changed files with 95 additions and 149 deletions.
diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -84,16 +84,6 @@ jobs:
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python-version }}
-    services:
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - uses: actions/checkout@v2
         with:

diff --git a/.gitignore b/.gitignore
@@ -54,7 +54,7 @@ vendor
 .terraform/
 *.tfvars
 
-# python 
+# python
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -107,6 +107,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+infra/scripts/*.conf
 
 # Translations
 *.mo

diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py
@@ -41,7 +41,7 @@
 
 try:
     from redis import Redis
-    from redis.cluster import ClusterNode, RedisCluster
+    from rediscluster import RedisCluster
 except ImportError as e:
     from feast.errors import FeastExtrasDependencyImportError
 
@@ -163,9 +163,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig):
                 online_store_config.connection_string
             )
             if online_store_config.redis_type == RedisType.redis_cluster:
-                kwargs["startup_nodes"] = [
-                    ClusterNode(**node) for node in startup_nodes
-                ]
+                kwargs["startup_nodes"] = startup_nodes
                 self._client = RedisCluster(**kwargs)
             else:
                 kwargs["host"] = startup_nodes[0]["host"]

diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
-#    pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt
+#    pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt setup.py
 #
 absl-py==1.0.0
     # via tensorflow-metadata
@@ -44,8 +44,6 @@ assertpy==1.1
     # via feast (setup.py)
 async-timeout==4.0.2
     # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
 attrs==21.4.0
     # via
     #   aiohttp
@@ -143,8 +141,6 @@ decorator==5.1.1
     #   ipython
 defusedxml==0.7.1
     # via nbconvert
-deprecated==1.2.13
-    # via redis
 deprecation==2.1.0
     # via testcontainers
 dill==0.3.4
@@ -282,18 +278,7 @@ idna==3.3
 imagesize==1.3.0
     # via sphinx
 importlib-metadata==4.2.0
-    # via
-    #   click
-    #   flake8
-    #   great-expectations
-    #   jsonschema
-    #   moto
-    #   pep517
-    #   pluggy
-    #   pre-commit
-    #   pytest
-    #   redis
-    #   virtualenv
+    # via great-expectations
 importlib-resources==5.4.0
     # via jsonschema
 iniconfig==1.1.1
@@ -358,10 +343,12 @@ jupyterlab-pygments==0.1.2
 jupyterlab-widgets==1.0.2
     # via ipywidgets
 libcst==0.4.1
-    # via google-cloud-datastore
+    # via
+    #   google-cloud-bigquery-storage
+    #   google-cloud-datastore
 locket==0.2.1
     # via partd
-markupsafe==2.1.0
+markupsafe==2.0.1
     # via
     #   jinja2
     #   moto
@@ -450,7 +437,6 @@ packaging==21.3
     #   google-cloud-bigquery
     #   google-cloud-firestore
     #   pytest
-    #   redis
     #   sphinx
 pandas==1.3.5
     # via
@@ -621,7 +607,11 @@ pyzmq==22.3.0
     # via
     #   jupyter-client
     #   notebook
-redis==4.1.4
+redis==3.5.3
+    # via
+    #   feast (setup.py)
+    #   redis-py-cluster
+redis-py-cluster==2.1.3
     # via feast (setup.py)
 regex==2022.1.18
     # via black
@@ -762,10 +752,10 @@ traitlets==5.1.1
     #   nbformat
     #   notebook
 typed-ast==1.5.2
-    # via
-    #   black
-    #   mypy
-types-protobuf==3.19.12
+    # via black
+types-futures==3.3.8
+    # via types-protobuf
+types-protobuf==3.19.2
     # via
     #   feast (setup.py)
     #   mypy-protobuf
@@ -787,22 +777,11 @@ types-urllib3==1.26.9
     # via types-requests
 typing-extensions==4.1.1
     # via
-    #   aiohttp
-    #   anyio
-    #   argon2-cffi
-    #   asgiref
-    #   async-timeout
     #   great-expectations
-    #   h11
-    #   importlib-metadata
-    #   jsonschema
     #   libcst
     #   mypy
     #   pydantic
-    #   starlette
     #   typing-inspect
-    #   uvicorn
-    #   yarl
 typing-inspect==0.7.1
     # via libcst
 tzdata==2021.5
@@ -841,9 +820,7 @@ wheel==0.37.1
 widgetsnbextension==3.5.2
     # via ipywidgets
 wrapt==1.13.3
-    # via
-    #   deprecated
-    #   testcontainers
+    # via testcontainers
 xmltodict==0.12.0
     # via moto
 yarl==1.7.2
@@ -852,7 +829,6 @@ zipp==3.7.0
     # via
     #   importlib-metadata
     #   importlib-resources
-    #   pep517
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt
+#    pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt setup.py
 #
 absl-py==1.0.0
     # via tensorflow-metadata
@@ -44,8 +44,6 @@ assertpy==1.1
     # via feast (setup.py)
 async-timeout==4.0.2
     # via aiohttp
-asynctest==0.13.0
-    # via aiohttp
 attrs==21.4.0
     # via
     #   aiohttp
@@ -116,6 +114,8 @@ click==8.0.3
     #   great-expectations
     #   pip-tools
     #   uvicorn
+cloudpickle==2.0.0
+    # via dask
 colorama==0.4.4
     # via feast (setup.py)
 coverage[toml]==6.3
@@ -131,6 +131,8 @@ cryptography==3.3.2
     #   pyjwt
     #   pyopenssl
     #   snowflake-connector-python
+dask==2022.1.1
+    # via feast (setup.py)
 debugpy==1.5.1
     # via ipykernel
 decorator==5.1.1
@@ -139,8 +141,6 @@ decorator==5.1.1
     #   ipython
 defusedxml==0.7.1
     # via nbconvert
-deprecated==1.2.13
-    # via redis
 deprecation==2.1.0
     # via testcontainers
 dill==0.3.4
@@ -181,6 +181,7 @@ frozenlist==1.3.0
 fsspec==2022.1.0
     # via
     #   adlfs
+    #   dask
     #   gcsfs
 gcsfs==2022.1.0
     # via feast (setup.py)
@@ -277,18 +278,7 @@ idna==3.3
 imagesize==1.3.0
     # via sphinx
 importlib-metadata==4.2.0
-    # via
-    #   click
-    #   flake8
-    #   great-expectations
-    #   jsonschema
-    #   moto
-    #   pep517
-    #   pluggy
-    #   pre-commit
-    #   pytest
-    #   redis
-    #   virtualenv
+    # via great-expectations
 importlib-resources==5.4.0
     # via jsonschema
 iniconfig==1.1.1
@@ -356,6 +346,8 @@ libcst==0.4.1
     # via
     #   google-cloud-bigquery-storage
     #   google-cloud-datastore
+locket==0.2.1
+    # via partd
 markupsafe==2.0.1
     # via
     #   jinja2
@@ -439,12 +431,12 @@ oscrypto==1.2.1
 packaging==21.3
     # via
     #   bleach
+    #   dask
     #   deprecation
     #   google-api-core
     #   google-cloud-bigquery
     #   google-cloud-firestore
     #   pytest
-    #   redis
     #   sphinx
 pandas==1.3.5
     # via
@@ -459,6 +451,8 @@ pandocfilters==1.5.0
     # via nbconvert
 parso==0.8.3
     # via jedi
+partd==1.2.0
+    # via dask
 pathspec==0.9.0
     # via black
 pbr==5.8.0
@@ -604,6 +598,7 @@ pytz-deprecation-shim==0.1.0.post0
     # via tzlocal
 pyyaml==6.0
     # via
+    #   dask
     #   feast (setup.py)
     #   libcst
     #   pre-commit
@@ -612,7 +607,11 @@ pyzmq==22.3.0
     # via
     #   jupyter-client
     #   notebook
-redis==4.1.2
+redis==3.5.3
+    # via
+    #   feast (setup.py)
+    #   redis-py-cluster
+redis-py-cluster==2.1.3
     # via feast (setup.py)
 regex==2022.1.18
     # via black
@@ -727,7 +726,10 @@ tomli==2.0.0
     #   mypy
     #   pep517
 toolz==0.11.2
-    # via altair
+    # via
+    #   altair
+    #   dask
+    #   partd
 tornado==6.1
     # via
     #   ipykernel
@@ -751,9 +753,7 @@ traitlets==5.1.1
     #   nbformat
     #   notebook
 typed-ast==1.5.2
-    # via
-    #   black
-    #   mypy
+    # via black
 types-futures==3.3.8
     # via types-protobuf
 types-protobuf==3.19.7
@@ -778,22 +778,11 @@ types-urllib3==1.26.8
     # via types-requests
 typing-extensions==4.0.1
     # via
-    #   aiohttp
-    #   anyio
-    #   argon2-cffi
-    #   asgiref
-    #   async-timeout
     #   great-expectations
-    #   h11
-    #   importlib-metadata
-    #   jsonschema
     #   libcst
     #   mypy
     #   pydantic
-    #   starlette
     #   typing-inspect
-    #   uvicorn
-    #   yarl
 typing-inspect==0.7.1
     # via libcst
 tzdata==2021.5
@@ -832,9 +821,7 @@ wheel==0.37.1
 widgetsnbextension==3.5.2
     # via ipywidgets
 wrapt==1.13.3
-    # via
-    #   deprecated
-    #   testcontainers
+    # via testcontainers
 xmltodict==0.12.0
     # via moto
 yarl==1.7.2
@@ -843,7 +830,6 @@ zipp==3.7.0
     # via
     #   importlib-metadata
     #   importlib-resources
-    #   pep517
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip