Skip to content

Commit

Permalink
fix: Rollback Redis-py to Redis-py-cluster (feast-dev#2347)
Browse files Browse the repository at this point in the history
* Roll back redis config and add tests back

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* lint

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix autogenerated import

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix lint

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Add redis-py-cluster back

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix dask ci

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Regenerate using respective python 3 versions

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Change test_entity_ttl test

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix lint

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Change integration workflow

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Temporarily disable for testing

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Continue making work

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Continue making work

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix workflow

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix workflow

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Update port numbers

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Update pr integration tests

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Reenable rest of tests

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Update data source creator

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Fix integration test

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Clean up code

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>

* Somehow removed the key components -,-

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>
  • Loading branch information
kevjumba authored Mar 3, 2022
1 parent c2a6c6c commit 1ba86fb
Show file tree
Hide file tree
Showing 10 changed files with 95 additions and 149 deletions.
10 changes: 0 additions & 10 deletions .github/workflows/pr_integration_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,6 @@ jobs:
env:
OS: ${{ matrix.os }}
PYTHON: ${{ matrix.python-version }}
services:
redis:
image: redis
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
with:
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ vendor
.terraform/
*.tfvars

# python
# python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
Expand Down Expand Up @@ -107,6 +107,7 @@ coverage.xml
*.cover
.hypothesis/
.pytest_cache/
infra/scripts/*.conf

# Translations
*.mo
Expand Down
6 changes: 2 additions & 4 deletions sdk/python/feast/infra/online_stores/redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@

try:
from redis import Redis
from redis.cluster import ClusterNode, RedisCluster
from rediscluster import RedisCluster
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError

Expand Down Expand Up @@ -163,9 +163,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig):
online_store_config.connection_string
)
if online_store_config.redis_type == RedisType.redis_cluster:
kwargs["startup_nodes"] = [
ClusterNode(**node) for node in startup_nodes
]
kwargs["startup_nodes"] = startup_nodes
self._client = RedisCluster(**kwargs)
else:
kwargs["host"] = startup_nodes[0]["host"]
Expand Down
56 changes: 16 additions & 40 deletions sdk/python/requirements/py3.7-ci-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with python 3.7
# To update, run:
#
# pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt
# pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt setup.py
#
absl-py==1.0.0
# via tensorflow-metadata
Expand Down Expand Up @@ -44,8 +44,6 @@ assertpy==1.1
# via feast (setup.py)
async-timeout==4.0.2
# via aiohttp
asynctest==0.13.0
# via aiohttp
attrs==21.4.0
# via
# aiohttp
Expand Down Expand Up @@ -143,8 +141,6 @@ decorator==5.1.1
# ipython
defusedxml==0.7.1
# via nbconvert
deprecated==1.2.13
# via redis
deprecation==2.1.0
# via testcontainers
dill==0.3.4
Expand Down Expand Up @@ -282,18 +278,7 @@ idna==3.3
imagesize==1.3.0
# via sphinx
importlib-metadata==4.2.0
# via
# click
# flake8
# great-expectations
# jsonschema
# moto
# pep517
# pluggy
# pre-commit
# pytest
# redis
# virtualenv
# via great-expectations
importlib-resources==5.4.0
# via jsonschema
iniconfig==1.1.1
Expand Down Expand Up @@ -358,10 +343,12 @@ jupyterlab-pygments==0.1.2
jupyterlab-widgets==1.0.2
# via ipywidgets
libcst==0.4.1
# via google-cloud-datastore
# via
# google-cloud-bigquery-storage
# google-cloud-datastore
locket==0.2.1
# via partd
markupsafe==2.1.0
markupsafe==2.0.1
# via
# jinja2
# moto
Expand Down Expand Up @@ -450,7 +437,6 @@ packaging==21.3
# google-cloud-bigquery
# google-cloud-firestore
# pytest
# redis
# sphinx
pandas==1.3.5
# via
Expand Down Expand Up @@ -621,7 +607,11 @@ pyzmq==22.3.0
# via
# jupyter-client
# notebook
redis==4.1.4
redis==3.5.3
# via
# feast (setup.py)
# redis-py-cluster
redis-py-cluster==2.1.3
# via feast (setup.py)
regex==2022.1.18
# via black
Expand Down Expand Up @@ -762,10 +752,10 @@ traitlets==5.1.1
# nbformat
# notebook
typed-ast==1.5.2
# via
# black
# mypy
types-protobuf==3.19.12
# via black
types-futures==3.3.8
# via types-protobuf
types-protobuf==3.19.2
# via
# feast (setup.py)
# mypy-protobuf
Expand All @@ -787,22 +777,11 @@ types-urllib3==1.26.9
# via types-requests
typing-extensions==4.1.1
# via
# aiohttp
# anyio
# argon2-cffi
# asgiref
# async-timeout
# great-expectations
# h11
# importlib-metadata
# jsonschema
# libcst
# mypy
# pydantic
# starlette
# typing-inspect
# uvicorn
# yarl
typing-inspect==0.7.1
# via libcst
tzdata==2021.5
Expand Down Expand Up @@ -841,9 +820,7 @@ wheel==0.37.1
widgetsnbextension==3.5.2
# via ipywidgets
wrapt==1.13.3
# via
# deprecated
# testcontainers
# via testcontainers
xmltodict==0.12.0
# via moto
yarl==1.7.2
Expand All @@ -852,7 +829,6 @@ zipp==3.7.0
# via
# importlib-metadata
# importlib-resources
# pep517

# The following packages are considered to be unsafe in a requirements file:
# pip
Expand Down
62 changes: 24 additions & 38 deletions sdk/python/requirements/py3.8-ci-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with python 3.8
# To update, run:
#
# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt
# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt setup.py
#
absl-py==1.0.0
# via tensorflow-metadata
Expand Down Expand Up @@ -44,8 +44,6 @@ assertpy==1.1
# via feast (setup.py)
async-timeout==4.0.2
# via aiohttp
asynctest==0.13.0
# via aiohttp
attrs==21.4.0
# via
# aiohttp
Expand Down Expand Up @@ -116,6 +114,8 @@ click==8.0.3
# great-expectations
# pip-tools
# uvicorn
cloudpickle==2.0.0
# via dask
colorama==0.4.4
# via feast (setup.py)
coverage[toml]==6.3
Expand All @@ -131,6 +131,8 @@ cryptography==3.3.2
# pyjwt
# pyopenssl
# snowflake-connector-python
dask==2022.1.1
# via feast (setup.py)
debugpy==1.5.1
# via ipykernel
decorator==5.1.1
Expand All @@ -139,8 +141,6 @@ decorator==5.1.1
# ipython
defusedxml==0.7.1
# via nbconvert
deprecated==1.2.13
# via redis
deprecation==2.1.0
# via testcontainers
dill==0.3.4
Expand Down Expand Up @@ -181,6 +181,7 @@ frozenlist==1.3.0
fsspec==2022.1.0
# via
# adlfs
# dask
# gcsfs
gcsfs==2022.1.0
# via feast (setup.py)
Expand Down Expand Up @@ -277,18 +278,7 @@ idna==3.3
imagesize==1.3.0
# via sphinx
importlib-metadata==4.2.0
# via
# click
# flake8
# great-expectations
# jsonschema
# moto
# pep517
# pluggy
# pre-commit
# pytest
# redis
# virtualenv
# via great-expectations
importlib-resources==5.4.0
# via jsonschema
iniconfig==1.1.1
Expand Down Expand Up @@ -356,6 +346,8 @@ libcst==0.4.1
# via
# google-cloud-bigquery-storage
# google-cloud-datastore
locket==0.2.1
# via partd
markupsafe==2.0.1
# via
# jinja2
Expand Down Expand Up @@ -439,12 +431,12 @@ oscrypto==1.2.1
packaging==21.3
# via
# bleach
# dask
# deprecation
# google-api-core
# google-cloud-bigquery
# google-cloud-firestore
# pytest
# redis
# sphinx
pandas==1.3.5
# via
Expand All @@ -459,6 +451,8 @@ pandocfilters==1.5.0
# via nbconvert
parso==0.8.3
# via jedi
partd==1.2.0
# via dask
pathspec==0.9.0
# via black
pbr==5.8.0
Expand Down Expand Up @@ -604,6 +598,7 @@ pytz-deprecation-shim==0.1.0.post0
# via tzlocal
pyyaml==6.0
# via
# dask
# feast (setup.py)
# libcst
# pre-commit
Expand All @@ -612,7 +607,11 @@ pyzmq==22.3.0
# via
# jupyter-client
# notebook
redis==4.1.2
redis==3.5.3
# via
# feast (setup.py)
# redis-py-cluster
redis-py-cluster==2.1.3
# via feast (setup.py)
regex==2022.1.18
# via black
Expand Down Expand Up @@ -727,7 +726,10 @@ tomli==2.0.0
# mypy
# pep517
toolz==0.11.2
# via altair
# via
# altair
# dask
# partd
tornado==6.1
# via
# ipykernel
Expand All @@ -751,9 +753,7 @@ traitlets==5.1.1
# nbformat
# notebook
typed-ast==1.5.2
# via
# black
# mypy
# via black
types-futures==3.3.8
# via types-protobuf
types-protobuf==3.19.7
Expand All @@ -778,22 +778,11 @@ types-urllib3==1.26.8
# via types-requests
typing-extensions==4.0.1
# via
# aiohttp
# anyio
# argon2-cffi
# asgiref
# async-timeout
# great-expectations
# h11
# importlib-metadata
# jsonschema
# libcst
# mypy
# pydantic
# starlette
# typing-inspect
# uvicorn
# yarl
typing-inspect==0.7.1
# via libcst
tzdata==2021.5
Expand Down Expand Up @@ -832,9 +821,7 @@ wheel==0.37.1
widgetsnbextension==3.5.2
# via ipywidgets
wrapt==1.13.3
# via
# deprecated
# testcontainers
# via testcontainers
xmltodict==0.12.0
# via moto
yarl==1.7.2
Expand All @@ -843,7 +830,6 @@ zipp==3.7.0
# via
# importlib-metadata
# importlib-resources
# pep517

# The following packages are considered to be unsafe in a requirements file:
# pip
Expand Down
Loading

0 comments on commit 1ba86fb

Please sign in to comment.