From 87d59fe92ecb3e47ec002d26fd09651fecbfd642 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 13:04:49 -0400 Subject: [PATCH 01/14] rebase Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 2 + sdk/python/feast/feature_store.py | 8 ++ .../infra/offline_stores/offline_store.py | 6 ++ .../feast/infra/online_stores/dynamodb.py | 74 ++++++++++++++----- .../feast/infra/online_stores/online_store.py | 6 ++ .../feast/infra/passthrough_provider.py | 8 ++ sdk/python/feast/infra/provider.py | 8 ++ sdk/python/tests/foo_provider.py | 6 ++ 8 files changed, 100 insertions(+), 18 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index b4ed591b04..e6e42718ab 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -101,8 +101,10 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): async_refresh() + await store.initialize() yield stop_refresh() + await store.close() app = FastAPI(lifespan=lifespan) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f9fa0a7881..21dec7b06f 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -2157,6 +2157,14 @@ def list_saved_datasets( self.project, allow_cache=allow_cache, tags=tags ) + async def initialize(self) -> None: + """Initialize long-lived clients and/or resources needed for accessing datastores""" + await self._get_provider().initialize(self.config) + + async def close(self) -> None: + """Cleanup any long-lived clients and/or resources""" + await self._get_provider().close() + def _print_materialization_log( start_date, end_date, num_feature_views: int, online_store: str diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 69d6bb278b..b3bd61b95f 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -388,3 +388,9 @@ def get_table_column_names_and_types_from_data_source( data_source: DataSource object """ return data_source.get_table_column_names_and_types(config=config) + + async def initialize(self, config: RepoConfig) -> None: + pass + + async def close(self) -> None: + pass diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index a915d2ee34..a97e81bc44 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio +import contextlib import itertools import logging from datetime import datetime from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from aiobotocore.config import AioConfig from pydantic import StrictBool, StrictStr from feast import Entity, FeatureView, utils @@ -75,6 +77,9 @@ class DynamoDBOnlineStoreConfig(FeastConfigBaseModel): session_based_auth: bool = False """AWS session based client authentication""" + max_pool_connections: int = 10 + """Max number of connections for async Dynamodb operations""" + class DynamoDBOnlineStore(OnlineStore): """ @@ -87,7 +92,14 @@ class DynamoDBOnlineStore(OnlineStore): _dynamodb_client = None _dynamodb_resource = None - _aioboto_session = None + + async def initialize(self, config: RepoConfig): + await _get_aiodynamodb_client( + config.online_store.region, config.online_store.max_pool_connections + ) + + async def close(self): + await _aiodynamodb_close() @property def async_supported(self) -> SupportedAsyncMethods: @@ -326,15 +338,17 @@ def to_tbl_resp(raw_client_response): batches.append(batch) entity_id_batches.append(entity_id_batch) - async with self._get_aiodynamodb_client(online_config.region) as client: - response_batches = await asyncio.gather( - *[ - client.batch_get_item( - RequestItems=entity_id_batch, - ) - for entity_id_batch in entity_id_batches - ] - ) + client = await _get_aiodynamodb_client( + online_config.region, online_config.max_pool_connections + ) + response_batches = await asyncio.gather( + *[ + client.batch_get_item( + RequestItems=entity_id_batch, + ) + for entity_id_batch in entity_id_batches + ] + ) result_batches = [] for batch, response in zip(batches, response_batches): @@ -349,14 +363,6 @@ def to_tbl_resp(raw_client_response): return list(itertools.chain(*result_batches)) - def _get_aioboto_session(self): - if self._aioboto_session is None: - self._aioboto_session = session.get_session() - return self._aioboto_session - - def _get_aiodynamodb_client(self, region: str): - return self._get_aioboto_session().create_client("dynamodb", region_name=region) - def _get_dynamodb_client( self, region: str, @@ -489,6 +495,38 @@ def _to_client_batch_get_payload(online_config, table_name, batch): } +_aioboto_session = None +_aioboto_client = None + + +def _get_aioboto_session(): + global _aioboto_session + if _aioboto_session is None: + logger.debug("initializing the aiobotocore session") + _aioboto_session = session.get_session() + return _aioboto_session + + +async def _get_aiodynamodb_client(region: str, max_pool_connections: int): + global _aioboto_client + if _aioboto_client is None: + logger.debug("initializing the aiobotocore dynamodb client") + client_context = _get_aioboto_session().create_client( + "dynamodb", + region_name=region, + config=AioConfig(max_pool_connections=max_pool_connections), + ) + context_stack = contextlib.AsyncExitStack() + _aioboto_client = await context_stack.enter_async_context(client_context) + return _aioboto_client + + +async def _aiodynamodb_close(): + global _aioboto_client + if _aioboto_client: + await _aioboto_client.close() + + def _initialize_dynamodb_client( region: str, endpoint_url: Optional[str] = None, diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 15dd843ba8..789885f82b 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -422,3 +422,9 @@ def retrieve_online_documents( raise NotImplementedError( f"Online store {self.__class__.__name__} does not support online retrieval" ) + + async def initialize(self, config: RepoConfig) -> None: + pass + + async def close(self) -> None: + pass diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 9482b808a9..f59d457ec0 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -518,3 +518,11 @@ def get_table_column_names_and_types_from_data_source( return self.offline_store.get_table_column_names_and_types_from_data_source( config=config, data_source=data_source ) + + async def initialize(self, config: RepoConfig) -> None: + await self.online_store.initialize(config) + await self.offline_store.initialize(config) + + async def close(self) -> None: + await self.online_store.close() + await self.offline_store.close() diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 47b7c65ef0..8351f389ad 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -476,6 +476,14 @@ def get_table_column_names_and_types_from_data_source( """ pass + @abstractmethod + async def initialize(self, config: RepoConfig) -> None: + pass + + @abstractmethod + async def close(self) -> None: + pass + def get_provider(config: RepoConfig) -> Provider: if "." not in config.provider: diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index bc30d3ef88..570a6d4f8d 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -213,3 +213,9 @@ async def online_write_batch_async( progress: Optional[Callable[[int], Any]], ) -> None: pass + + async def initialize(self, config: RepoConfig) -> None: + pass + + async def close(self) -> None: + pass From 7793d99f3f39b834f87b3f3818757025031011f2 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Thu, 17 Oct 2024 15:40:23 -0400 Subject: [PATCH 02/14] offline store init doesnt make sense Signed-off-by: Rob Howley --- sdk/python/feast/infra/offline_stores/offline_store.py | 6 ------ sdk/python/feast/infra/passthrough_provider.py | 2 -- 2 files changed, 8 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index b3bd61b95f..69d6bb278b 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -388,9 +388,3 @@ def get_table_column_names_and_types_from_data_source( data_source: DataSource object """ return data_source.get_table_column_names_and_types(config=config) - - async def initialize(self, config: RepoConfig) -> None: - pass - - async def close(self) -> None: - pass diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index f59d457ec0..a1e9ef82ad 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -521,8 +521,6 @@ def get_table_column_names_and_types_from_data_source( async def initialize(self, config: RepoConfig) -> None: await self.online_store.initialize(config) - await self.offline_store.initialize(config) async def close(self) -> None: await self.online_store.close() - await self.offline_store.close() From f04711eb6c95f5c488d3598c133e819207ed1711 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 15:24:09 -0400 Subject: [PATCH 03/14] dont init or close Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index e6e42718ab..e5e7410715 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -101,10 +101,10 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): async_refresh() - await store.initialize() + # await store.initialize() yield stop_refresh() - await store.close() + # await store.close() app = FastAPI(lifespan=lifespan) From d1094adc039f80eb113a0d7613a1d32fcb68dc0e Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 16:13:02 -0400 Subject: [PATCH 04/14] update test to handle event loop for dynamo case Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index d08e1104eb..8a150d0cc9 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -1,3 +1,4 @@ +import asyncio import json from typing import List @@ -137,5 +138,7 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) + asyncio.run(fs.initalize()) client = TestClient(get_app(fs)) yield client + asyncio.run(fs.close()) From afaa4120107e9eddb3e9acd781a57021b5ceda4b Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 16:36:06 -0400 Subject: [PATCH 05/14] use run util complete Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 4 ++-- .../integration/online_store/test_python_feature_server.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index e5e7410715..e6e42718ab 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -101,10 +101,10 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): async_refresh() - # await store.initialize() + await store.initialize() yield stop_refresh() - # await store.close() + await store.close() app = FastAPI(lifespan=lifespan) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 8a150d0cc9..7e6c09c1de 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -138,7 +138,8 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) - asyncio.run(fs.initalize()) + loop = asyncio.get_event_loop() + loop.run_until_complete(fs.initalize()) client = TestClient(get_app(fs)) yield client - asyncio.run(fs.close()) + loop.run_until_complete(fs.close()) From 2245caa10e431a9d65af8ba14a3d0264df684900 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 17:05:14 -0400 Subject: [PATCH 06/14] fix: spelling sigh Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 7e6c09c1de..159660d20e 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -139,7 +139,7 @@ def python_fs_client(environment, universal_data_sources, request): fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) loop = asyncio.get_event_loop() - loop.run_until_complete(fs.initalize()) + loop.run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) yield client loop.run_until_complete(fs.close()) From eebf71de944c297c20c0cbfee74d47a2366838a8 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:28:26 -0400 Subject: [PATCH 07/14] run integration test as async since that is default for read Signed-off-by: Rob Howley --- sdk/python/feast/feature_server.py | 2 +- .../requirements/py3.10-ci-requirements.txt | 172 +++++++++++++--- .../requirements/py3.11-ci-requirements.txt | 191 ++++++++++++++++-- .../requirements/py3.9-ci-requirements.txt | 183 +++++++++++++---- .../test_python_feature_server.py | 9 +- 5 files changed, 471 insertions(+), 86 deletions(-) diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index e6e42718ab..f485d874e1 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -100,8 +100,8 @@ def async_refresh(): @asynccontextmanager async def lifespan(app: FastAPI): - async_refresh() await store.initialize() + async_refresh() yield stop_refresh() await store.close() diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 57a21cd6d9..0041e21703 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt aiobotocore==2.15.1 + # via feast (setup.py) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 @@ -21,6 +22,8 @@ anyio==4.5.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -30,6 +33,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -52,7 +56,9 @@ azure-core==1.31.0 # azure-identity # azure-storage-blob azure-identity==1.18.0 + # via feast (setup.py) azure-storage-blob==12.23.0 + # via feast (setup.py) babel==2.16.0 # via # jupyterlab-server @@ -60,10 +66,13 @@ babel==2.16.0 beautifulsoup4==4.12.3 # via nbconvert bigtree==0.21.1 + # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.35.23 - # via moto + # via + # feast (setup.py) + # moto botocore==1.35.23 # via # aiobotocore @@ -72,11 +81,13 @@ botocore==1.35.23 # s3transfer build==1.2.2 # via + # feast (setup.py) # pip-tools # singlestoredb cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 + # via feast (setup.py) certifi==2024.8.30 # via # elastic-transport @@ -99,6 +110,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -107,15 +119,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.1 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -130,7 +145,9 @@ cryptography==43.0.1 cython==3.0.11 # via thriftpy2 dask[dataframe]==2024.9.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.14 # via dask db-dtypes==1.3.0 @@ -142,9 +159,11 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.20.0 + # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.1.0 @@ -156,6 +175,7 @@ duckdb==1.1.0 elastic-transport==8.15.0 # via elasticsearch elasticsearch==8.15.1 + # via feast (setup.py) entrypoints==0.4 # via altair exceptiongroup==1.2.2 @@ -167,7 +187,10 @@ execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data +faiss-cpu==1.9.0 + # via feast (setup.py) fastapi==0.115.2 + # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat filelock==3.16.1 @@ -181,11 +204,14 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2024.9.0 - # via dask + # via + # feast (setup.py) + # dask geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.20.0 # via + # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -203,8 +229,11 @@ google-auth==2.35.0 # google-cloud-storage # kubernetes google-cloud-bigquery[pandas]==3.25.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.26.0 + # via feast (setup.py) google-cloud-bigtable==2.26.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -212,7 +241,9 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-storage google-cloud-datastore==2.20.1 + # via feast (setup.py) google-cloud-storage==2.18.2 + # via feast (setup.py) google-crc32c==1.6.0 # via # google-cloud-storage @@ -223,16 +254,17 @@ google-resumable-media==2.7.2 # google-cloud-storage googleapis-common-protos[grpc]==1.65.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.21 -greenlet==3.1.0 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.66.1 # via + # feast (setup.py) # google-api-core # googleapis-common-protos # grpc-google-iam-v1 @@ -242,30 +274,44 @@ grpcio==1.66.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.3 + # via feast (setup.py) grpcio-reflection==1.62.3 + # via feast (setup.py) grpcio-status==1.62.3 # via google-api-core grpcio-testing==1.62.3 + # via feast (setup.py) grpcio-tools==1.62.3 + # via feast (setup.py) gunicorn==23.0.0 + # via + # feast (setup.py) + # uvicorn-worker h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.5.0 + # via feast (setup.py) hiredis==2.4.0 + # via feast (setup.py) httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.2 # via + # feast (setup.py) # jupyterlab # python-keycloak ibis-framework[duckdb]==9.5.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.1 + # via feast (setup.py) identify==2.6.1 # via pre-commit idna==3.10 @@ -279,9 +325,7 @@ idna==3.10 imagesize==1.4.1 # via sphinx importlib-metadata==8.5.0 - # via - # build - # dask + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -301,6 +345,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # great-expectations # jupyter-server @@ -323,6 +368,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.23.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -370,6 +416,7 @@ jupyterlab-widgets==3.0.13 jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.4 @@ -390,13 +437,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==5.0.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.31.0 # via # azure-identity @@ -408,10 +459,13 @@ multidict==6.1.0 # aiohttp # yarl mypy==1.11.2 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -434,9 +488,11 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes + # faiss-cpu # great-expectations # ibis-framework # pandas @@ -452,6 +508,7 @@ packaging==24.1 # dask # db-dtypes # deprecation + # faiss-cpu # google-cloud-bigquery # great-expectations # gunicorn @@ -468,6 +525,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -493,6 +551,7 @@ pexpect==4.9.0 pip==24.2 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -505,8 +564,11 @@ ply==3.11 portalocker==2.10.1 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 - # via jupyter-server + # via + # feast (setup.py) + # jupyter-server prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -517,6 +579,7 @@ proto-plus==1.24.0 # google-cloud-datastore protobuf==4.25.5 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery-storage # google-cloud-bigtable @@ -532,9 +595,12 @@ protobuf==4.25.5 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg[binary, pool]==3.2.2 -psycopg-binary==3.2.2 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.3 # via psycopg @@ -545,12 +611,14 @@ ptyprocess==0.7.0 pure-eval==0.2.3 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==17.0.0 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -566,28 +634,35 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.9.2 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich # sphinx pyjwt[crypto]==2.9.0 # via + # feast (setup.py) # msal # singlestoredb # snowflake-connector-python pymssql==2.3.1 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.2.1 # via snowflake-connector-python pyparsing==3.1.4 @@ -597,8 +672,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.2 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -608,13 +685,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -632,6 +717,7 @@ python-dotenv==1.0.1 python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 + # via feast (setup.py) pytz==2024.2 # via # great-expectations @@ -641,6 +727,7 @@ pytz==2024.2 # trino pyyaml==6.0.2 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -654,15 +741,19 @@ pyzmq==26.2.0 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.9.11 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # docker # google-api-core @@ -708,6 +799,7 @@ ruamel-yaml==0.17.40 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.6.6 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.1 @@ -722,6 +814,7 @@ setuptools==75.1.0 # pip-tools # singlestoredb singlestoredb==1.6.3 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -742,11 +835,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.12.2 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -760,9 +855,11 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlalchemy[mypy]==2.0.35 + # via feast (setup.py) sqlglot==25.20.1 # via ibis-framework sqlite-vec==0.1.1 + # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb stack-data==0.6.3 @@ -772,17 +869,21 @@ starlette==0.40.0 substrait==0.23.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.5.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.2 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -810,7 +911,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.5 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -827,23 +930,37 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.329.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis types-python-dateutil==2.9.0.20240906 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.2.0.20240913 + # via feast (setup.py) types-pyyaml==6.0.12.20240917 + # via feast (setup.py) types-redis==4.6.0.20240903 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==75.1.0.20240917 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 @@ -879,6 +996,7 @@ uri-template==1.3.0 # via jsonschema urllib3==2.2.3 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -889,11 +1007,17 @@ urllib3==2.2.3 # responses # testcontainers uvicorn[standard]==0.30.6 -uvicorn-worker + # via + # feast (setup.py) + # uvicorn-worker +uvicorn-worker==0.2.0 + # via feast (setup.py) uvloop==0.20.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index ed6dc239d3..9e1225d9ed 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt aiobotocore==2.15.1 + # via feast (setup.py) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 @@ -21,6 +22,8 @@ anyio==4.5.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -30,6 +33,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -37,7 +41,9 @@ async-lru==2.0.4 async-property==0.2.2 # via python-keycloak async-timeout==4.0.3 - # via redis + # via + # aiohttp + # redis atpublic==5.0 # via ibis-framework attrs==24.2.0 @@ -50,7 +56,9 @@ azure-core==1.31.0 # azure-identity # azure-storage-blob azure-identity==1.18.0 + # via feast (setup.py) azure-storage-blob==12.23.0 + # via feast (setup.py) babel==2.16.0 # via # jupyterlab-server @@ -58,10 +66,13 @@ babel==2.16.0 beautifulsoup4==4.12.3 # via nbconvert bigtree==0.21.1 + # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.35.23 - # via moto + # via + # feast (setup.py) + # moto botocore==1.35.23 # via # aiobotocore @@ -70,11 +81,13 @@ botocore==1.35.23 # s3transfer build==1.2.2 # via + # feast (setup.py) # pip-tools # singlestoredb cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 + # via feast (setup.py) certifi==2024.8.30 # via # elastic-transport @@ -97,6 +110,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -105,15 +119,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.1 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -128,7 +145,9 @@ cryptography==43.0.1 cython==3.0.11 # via thriftpy2 dask[dataframe]==2024.9.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.14 # via dask db-dtypes==1.3.0 @@ -140,9 +159,11 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.20.0 + # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.1.0 @@ -154,13 +175,22 @@ duckdb==1.1.0 elastic-transport==8.15.0 # via elasticsearch elasticsearch==8.15.1 + # via feast (setup.py) entrypoints==0.4 # via altair +exceptiongroup==1.2.2 + # via + # anyio + # ipython + # pytest execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data +faiss-cpu==1.9.0 + # via feast (setup.py) fastapi==0.115.2 + # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat filelock==3.16.1 @@ -174,11 +204,14 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2024.9.0 - # via dask + # via + # feast (setup.py) + # dask geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.20.0 # via + # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -196,8 +229,11 @@ google-auth==2.35.0 # google-cloud-storage # kubernetes google-cloud-bigquery[pandas]==3.25.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.26.0 + # via feast (setup.py) google-cloud-bigtable==2.26.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -205,7 +241,9 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-storage google-cloud-datastore==2.20.1 + # via feast (setup.py) google-cloud-storage==2.18.2 + # via feast (setup.py) google-crc32c==1.6.0 # via # google-cloud-storage @@ -216,16 +254,17 @@ google-resumable-media==2.7.2 # google-cloud-storage googleapis-common-protos[grpc]==1.65.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.21 -greenlet==3.1.0 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.66.1 # via + # feast (setup.py) # google-api-core # googleapis-common-protos # grpc-google-iam-v1 @@ -235,30 +274,44 @@ grpcio==1.66.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.3 + # via feast (setup.py) grpcio-reflection==1.62.3 + # via feast (setup.py) grpcio-status==1.62.3 # via google-api-core grpcio-testing==1.62.3 + # via feast (setup.py) grpcio-tools==1.62.3 + # via feast (setup.py) gunicorn==23.0.0 + # via + # feast (setup.py) + # uvicorn-worker h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.5.0 + # via feast (setup.py) hiredis==2.4.0 + # via feast (setup.py) httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.2 # via + # feast (setup.py) # jupyterlab # python-keycloak ibis-framework[duckdb]==9.5.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.1 + # via feast (setup.py) identify==2.6.1 # via pre-commit idna==3.10 @@ -292,6 +345,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # great-expectations # jupyter-server @@ -314,6 +368,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.23.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -361,6 +416,7 @@ jupyterlab-widgets==3.0.13 jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.4 @@ -381,13 +437,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==5.0.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.31.0 # via # azure-identity @@ -399,10 +459,13 @@ multidict==6.1.0 # aiohttp # yarl mypy==1.11.2 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -425,9 +488,11 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes + # faiss-cpu # great-expectations # ibis-framework # pandas @@ -443,6 +508,7 @@ packaging==24.1 # dask # db-dtypes # deprecation + # faiss-cpu # google-cloud-bigquery # great-expectations # gunicorn @@ -459,6 +525,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -484,6 +551,7 @@ pexpect==4.9.0 pip==24.2 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -496,8 +564,11 @@ ply==3.11 portalocker==2.10.1 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 - # via jupyter-server + # via + # feast (setup.py) + # jupyter-server prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -508,6 +579,7 @@ proto-plus==1.24.0 # google-cloud-datastore protobuf==4.25.5 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery-storage # google-cloud-bigtable @@ -523,9 +595,12 @@ protobuf==4.25.5 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg[binary, pool]==3.2.2 -psycopg-binary==3.2.2 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.3 # via psycopg @@ -536,12 +611,14 @@ ptyprocess==0.7.0 pure-eval==0.2.3 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==17.0.0 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -557,28 +634,35 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.9.2 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich # sphinx pyjwt[crypto]==2.9.0 # via + # feast (setup.py) # msal # singlestoredb # snowflake-connector-python pymssql==2.3.1 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.2.1 # via snowflake-connector-python pyparsing==3.1.4 @@ -588,8 +672,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.2 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -599,13 +685,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -623,6 +717,7 @@ python-dotenv==1.0.1 python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 + # via feast (setup.py) pytz==2024.2 # via # great-expectations @@ -632,6 +727,7 @@ pytz==2024.2 # trino pyyaml==6.0.2 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -645,15 +741,19 @@ pyzmq==26.2.0 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.9.11 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # docker # google-api-core @@ -699,6 +799,7 @@ ruamel-yaml==0.17.40 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.6.6 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.14.1 @@ -713,6 +814,7 @@ setuptools==75.1.0 # pip-tools # singlestoredb singlestoredb==1.6.3 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -733,11 +835,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.12.2 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -751,9 +855,11 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlalchemy[mypy]==2.0.35 + # via feast (setup.py) sqlglot==25.20.1 # via ibis-framework sqlite-vec==0.1.1 + # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb stack-data==0.6.3 @@ -763,17 +869,31 @@ starlette==0.40.0 substrait==0.23.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.5.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.2 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) +tomli==2.0.2 + # via + # build + # coverage + # jupyterlab + # mypy + # pip-tools + # pytest + # pytest-env + # singlestoredb tomlkit==0.13.2 # via snowflake-connector-python toolz==0.12.1 @@ -791,7 +911,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.5 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -808,27 +930,43 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.329.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis types-python-dateutil==2.9.0.20240906 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.2.0.20240913 + # via feast (setup.py) types-pyyaml==6.0.12.20240917 + # via feast (setup.py) types-redis==4.6.0.20240903 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==75.1.0.20240917 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via + # anyio + # async-lru # azure-core # azure-identity # azure-storage-blob @@ -837,6 +975,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # multidict # mypy # psycopg # psycopg-pool @@ -846,6 +985,7 @@ typing-extensions==4.12.2 # sqlalchemy # testcontainers # typeguard + # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 @@ -856,6 +996,7 @@ uri-template==1.3.0 # via jsonschema urllib3==2.2.3 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -866,11 +1007,17 @@ urllib3==2.2.3 # responses # testcontainers uvicorn[standard]==0.30.6 -uvicorn-worker + # via + # feast (setup.py) + # uvicorn-worker +uvicorn-worker==0.2.0 + # via feast (setup.py) uvloop==0.20.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e7d6686b4d..e7989843a8 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,6 +1,7 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt +# uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt aiobotocore==2.15.1 + # via feast (setup.py) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 @@ -21,6 +22,8 @@ anyio==4.5.0 # jupyter-server # starlette # watchfiles +appnope==0.1.4 + # via ipykernel argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -30,6 +33,7 @@ arrow==1.3.0 asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 + # via feast (setup.py) asttokens==2.4.1 # via stack-data async-lru==2.0.4 @@ -52,7 +56,9 @@ azure-core==1.31.0 # azure-identity # azure-storage-blob azure-identity==1.18.0 + # via feast (setup.py) azure-storage-blob==12.23.0 + # via feast (setup.py) babel==2.16.0 # via # jupyterlab-server @@ -62,10 +68,13 @@ beautifulsoup4==4.12.3 bidict==0.23.1 # via ibis-framework bigtree==0.21.1 + # via feast (setup.py) bleach==6.1.0 # via nbconvert boto3==1.35.23 - # via moto + # via + # feast (setup.py) + # moto botocore==1.35.23 # via # aiobotocore @@ -74,11 +83,13 @@ botocore==1.35.23 # s3transfer build==1.2.2 # via + # feast (setup.py) # pip-tools # singlestoredb cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 + # via feast (setup.py) certifi==2024.8.30 # via # elastic-transport @@ -101,6 +112,7 @@ charset-normalizer==3.3.2 # snowflake-connector-python click==8.1.7 # via + # feast (setup.py) # dask # geomet # great-expectations @@ -109,15 +121,18 @@ click==8.1.7 cloudpickle==3.0.0 # via dask colorama==0.4.6 - # via great-expectations + # via + # feast (setup.py) + # great-expectations comm==0.2.2 # via # ipykernel # ipywidgets coverage[toml]==7.6.1 # via pytest-cov -cryptography==43.0.1 +cryptography==42.0.8 # via + # feast (setup.py) # azure-identity # azure-storage-blob # great-expectations @@ -132,7 +147,9 @@ cryptography==43.0.1 cython==3.0.11 # via thriftpy2 dask[dataframe]==2024.8.0 - # via dask-expr + # via + # feast (setup.py) + # dask-expr dask-expr==1.1.10 # via dask db-dtypes==1.3.0 @@ -144,9 +161,11 @@ decorator==5.1.1 defusedxml==0.7.1 # via nbconvert deltalake==0.20.0 + # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.8 + # via feast (setup.py) distlib==0.3.8 # via virtualenv docker==7.1.0 @@ -158,6 +177,7 @@ duckdb==0.10.3 elastic-transport==8.15.0 # via elasticsearch elasticsearch==8.15.1 + # via feast (setup.py) entrypoints==0.4 # via altair exceptiongroup==1.2.2 @@ -169,7 +189,10 @@ execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data +faiss-cpu==1.9.0 + # via feast (setup.py) fastapi==0.115.2 + # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat filelock==3.16.1 @@ -183,11 +206,14 @@ frozenlist==1.4.1 # aiohttp # aiosignal fsspec==2024.9.0 - # via dask + # via + # feast (setup.py) + # dask geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.20.0 # via + # feast (setup.py) # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-bigtable @@ -205,8 +231,11 @@ google-auth==2.35.0 # google-cloud-storage # kubernetes google-cloud-bigquery[pandas]==3.25.0 + # via feast (setup.py) google-cloud-bigquery-storage==2.26.0 + # via feast (setup.py) google-cloud-bigtable==2.26.0 + # via feast (setup.py) google-cloud-core==2.4.1 # via # google-cloud-bigquery @@ -214,7 +243,9 @@ google-cloud-core==2.4.1 # google-cloud-datastore # google-cloud-storage google-cloud-datastore==2.20.1 + # via feast (setup.py) google-cloud-storage==2.18.2 + # via feast (setup.py) google-crc32c==1.6.0 # via # google-cloud-storage @@ -225,16 +256,17 @@ google-resumable-media==2.7.2 # google-cloud-storage googleapis-common-protos[grpc]==1.65.0 # via + # feast (setup.py) # google-api-core # grpc-google-iam-v1 # grpcio-status great-expectations==0.18.21 -greenlet==3.1.0 - # via sqlalchemy + # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable grpcio==1.66.1 # via + # feast (setup.py) # google-api-core # googleapis-common-protos # grpc-google-iam-v1 @@ -244,30 +276,44 @@ grpcio==1.66.1 # grpcio-testing # grpcio-tools grpcio-health-checking==1.62.3 + # via feast (setup.py) grpcio-reflection==1.62.3 + # via feast (setup.py) grpcio-status==1.62.3 # via google-api-core grpcio-testing==1.62.3 + # via feast (setup.py) grpcio-tools==1.62.3 + # via feast (setup.py) gunicorn==23.0.0 + # via + # feast (setup.py) + # uvicorn-worker h11==0.14.0 # via # httpcore # uvicorn happybase==1.2.0 + # via feast (setup.py) hazelcast-python-client==5.5.0 + # via feast (setup.py) hiredis==2.4.0 + # via feast (setup.py) httpcore==1.0.5 # via httpx httptools==0.6.1 # via uvicorn httpx==0.27.2 # via + # feast (setup.py) # jupyterlab # python-keycloak ibis-framework[duckdb]==9.0.0 - # via ibis-substrait + # via + # feast (setup.py) + # ibis-substrait ibis-substrait==4.0.1 + # via feast (setup.py) identify==2.6.1 # via pre-commit idna==3.10 @@ -281,16 +327,7 @@ idna==3.10 imagesize==1.4.1 # via sphinx importlib-metadata==8.5.0 - # via - # build - # dask - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # nbconvert - # sphinx - # typeguard + # via dask iniconfig==2.0.0 # via pytest ipykernel==6.29.5 @@ -310,6 +347,7 @@ jedi==0.19.1 # via ipython jinja2==3.1.4 # via + # feast (setup.py) # altair # great-expectations # jupyter-server @@ -332,6 +370,7 @@ jsonpointer==3.0.0 # jsonschema jsonschema[format-nongpl]==4.23.0 # via + # feast (setup.py) # altair # great-expectations # jupyter-events @@ -379,6 +418,7 @@ jupyterlab-widgets==3.0.13 jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 + # via feast (setup.py) locket==1.0.0 # via partd makefun==1.15.4 @@ -399,13 +439,17 @@ matplotlib-inline==0.1.7 mdurl==0.1.2 # via markdown-it-py minio==7.1.0 + # via feast (setup.py) mistune==3.0.2 # via # great-expectations # nbconvert mmh3==5.0.0 + # via feast (setup.py) mock==2.0.0 + # via feast (setup.py) moto==4.2.14 + # via feast (setup.py) msal==1.31.0 # via # azure-identity @@ -417,10 +461,13 @@ multidict==6.1.0 # aiohttp # yarl mypy==1.11.2 - # via sqlalchemy + # via + # feast (setup.py) + # sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 + # via feast (setup.py) nbclient==0.10.0 # via nbconvert nbconvert==7.16.4 @@ -443,9 +490,11 @@ notebook-shim==0.2.4 # notebook numpy==1.26.4 # via + # feast (setup.py) # altair # dask # db-dtypes + # faiss-cpu # great-expectations # ibis-framework # pandas @@ -461,6 +510,7 @@ packaging==24.1 # dask # db-dtypes # deprecation + # faiss-cpu # google-cloud-bigquery # great-expectations # gunicorn @@ -476,6 +526,7 @@ packaging==24.1 # sphinx pandas==2.2.2 # via + # feast (setup.py) # altair # dask # dask-expr @@ -501,6 +552,7 @@ pexpect==4.9.0 pip==24.2 # via pip-tools pip-tools==7.4.1 + # via feast (setup.py) platformdirs==3.11.0 # via # jupyter-core @@ -513,8 +565,11 @@ ply==3.11 portalocker==2.10.1 # via msal-extensions pre-commit==3.3.1 + # via feast (setup.py) prometheus-client==0.20.0 - # via jupyter-server + # via + # feast (setup.py) + # jupyter-server prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -525,6 +580,7 @@ proto-plus==1.24.0 # google-cloud-datastore protobuf==4.25.5 # via + # feast (setup.py) # google-api-core # google-cloud-bigquery-storage # google-cloud-bigtable @@ -540,9 +596,12 @@ protobuf==4.25.5 # proto-plus # substrait psutil==5.9.0 - # via ipykernel -psycopg[binary, pool]==3.2.2 -psycopg-binary==3.2.2 + # via + # feast (setup.py) + # ipykernel +psycopg[binary, pool]==3.1.19 + # via feast (setup.py) +psycopg-binary==3.1.19 # via psycopg psycopg-pool==3.2.3 # via psycopg @@ -553,12 +612,14 @@ ptyprocess==0.7.0 pure-eval==0.2.3 # via stack-data py==1.11.0 + # via feast (setup.py) py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark pyarrow==16.1.0 # via + # feast (setup.py) # dask-expr # db-dtypes # deltalake @@ -574,28 +635,35 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 + # via feast (setup.py) pycparser==2.22 # via cffi pydantic==2.9.2 # via + # feast (setup.py) # fastapi # great-expectations pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via + # feast (setup.py) # ipython # nbconvert # rich # sphinx pyjwt[crypto]==2.9.0 # via + # feast (setup.py) # msal # singlestoredb # snowflake-connector-python pymssql==2.3.1 + # via feast (setup.py) pymysql==1.1.1 + # via feast (setup.py) pyodbc==5.1.0 + # via feast (setup.py) pyopenssl==24.2.1 # via snowflake-connector-python pyparsing==3.1.4 @@ -605,8 +673,10 @@ pyproject-hooks==1.1.0 # build # pip-tools pyspark==3.5.2 + # via feast (setup.py) pytest==7.4.4 # via + # feast (setup.py) # pytest-benchmark # pytest-cov # pytest-env @@ -616,13 +686,21 @@ pytest==7.4.4 # pytest-timeout # pytest-xdist pytest-benchmark==3.4.1 + # via feast (setup.py) pytest-cov==5.0.0 + # via feast (setup.py) pytest-env==1.1.3 + # via feast (setup.py) pytest-lazy-fixture==0.6.3 + # via feast (setup.py) pytest-mock==1.10.4 + # via feast (setup.py) pytest-ordering==0.6 + # via feast (setup.py) pytest-timeout==1.4.2 + # via feast (setup.py) pytest-xdist==3.6.1 + # via feast (setup.py) python-dateutil==2.9.0.post0 # via # arrow @@ -640,6 +718,7 @@ python-dotenv==1.0.1 python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 + # via feast (setup.py) pytz==2024.2 # via # great-expectations @@ -649,6 +728,7 @@ pytz==2024.2 # trino pyyaml==6.0.2 # via + # feast (setup.py) # dask # ibis-substrait # jupyter-events @@ -662,15 +742,19 @@ pyzmq==26.2.0 # jupyter-client # jupyter-server redis==4.6.0 + # via feast (setup.py) referencing==0.35.1 # via # jsonschema # jsonschema-specifications # jupyter-events regex==2024.9.11 - # via parsimonious + # via + # feast (setup.py) + # parsimonious requests==2.32.3 # via + # feast (setup.py) # azure-core # docker # google-api-core @@ -716,6 +800,7 @@ ruamel-yaml==0.17.40 ruamel-yaml-clib==0.2.8 # via ruamel-yaml ruff==0.6.6 + # via feast (setup.py) s3transfer==0.10.2 # via boto3 scipy==1.13.1 @@ -730,6 +815,7 @@ setuptools==75.1.0 # pip-tools # singlestoredb singlestoredb==1.6.3 + # via feast (setup.py) six==1.16.0 # via # asttokens @@ -750,11 +836,13 @@ sniffio==1.3.1 snowballstemmer==2.2.0 # via sphinx snowflake-connector-python[pandas]==3.12.2 + # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 sphinx==6.2.1 + # via feast (setup.py) sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -768,9 +856,11 @@ sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 # via sphinx sqlalchemy[mypy]==2.0.35 + # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework sqlite-vec==0.1.1 + # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb stack-data==0.6.3 @@ -780,17 +870,21 @@ starlette==0.40.0 substrait==0.23.0 # via ibis-substrait tabulate==0.9.0 + # via feast (setup.py) tenacity==8.5.0 + # via feast (setup.py) terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals testcontainers==4.4.0 + # via feast (setup.py) thriftpy2==0.5.2 # via happybase tinycss2==1.3.0 # via nbconvert toml==0.10.2 + # via feast (setup.py) tomli==2.0.1 # via # build @@ -818,7 +912,9 @@ tornado==6.4.1 # notebook # terminado tqdm==4.66.5 - # via great-expectations + # via + # feast (setup.py) + # great-expectations traitlets==5.14.3 # via # comm @@ -835,28 +931,41 @@ traitlets==5.14.3 # nbconvert # nbformat trino==0.329.0 + # via feast (setup.py) typeguard==4.3.0 + # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl types-protobuf==3.19.22 - # via mypy-protobuf + # via + # feast (setup.py) + # mypy-protobuf types-pymysql==1.1.0.20240524 + # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis types-python-dateutil==2.9.0.20240906 - # via arrow + # via + # feast (setup.py) + # arrow types-pytz==2024.2.0.20240913 + # via feast (setup.py) types-pyyaml==6.0.12.20240917 + # via feast (setup.py) types-redis==4.6.0.20240903 + # via feast (setup.py) types-requests==2.30.0.0 + # via feast (setup.py) types-setuptools==75.1.0.20240917 - # via types-cffi + # via + # feast (setup.py) + # types-cffi types-tabulate==0.9.0.20240106 + # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via - # aioitertools # anyio # async-lru # azure-core @@ -865,7 +974,6 @@ typing-extensions==4.12.2 # fastapi # great-expectations # ibis-framework - # ipython # jwcrypto # multidict # mypy @@ -875,7 +983,6 @@ typing-extensions==4.12.2 # pydantic-core # snowflake-connector-python # sqlalchemy - # starlette # testcontainers # typeguard # uvicorn @@ -889,6 +996,7 @@ uri-template==1.3.0 # via jsonschema urllib3==1.26.20 # via + # feast (setup.py) # botocore # docker # elastic-transport @@ -897,14 +1005,19 @@ urllib3==1.26.20 # minio # requests # responses - # snowflake-connector-python # testcontainers uvicorn[standard]==0.30.6 -uvicorn-worker + # via + # feast (setup.py) + # uvicorn-worker +uvicorn-worker==0.2.0 + # via feast (setup.py) uvloop==0.20.0 # via uvicorn virtualenv==20.23.0 - # via pre-commit + # via + # feast (setup.py) + # pre-commit watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 159660d20e..f86d87ff2d 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -59,8 +59,9 @@ def test_get_online_features(python_fs_client): @pytest.mark.integration @pytest.mark.universal_online_stores -def test_push(python_fs_client): - initial_temp = _get_temperatures_from_feature_server( +@pytest.mark.asyncio +async def test_push(python_fs_client): + initial_temp = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] )[0] json_data = json.dumps( @@ -81,7 +82,7 @@ def test_push(python_fs_client): # Check new pushed temperature is fetched assert response.status_code == 200 - assert _get_temperatures_from_feature_server( + assert await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] ) == [initial_temp * 100] @@ -112,7 +113,7 @@ def test_push_source_does_not_exist(python_fs_client): ) -def _get_temperatures_from_feature_server(client, location_ids: List[int]): +async def _get_temperatures_from_feature_server(client, location_ids: List[int]): get_request_data = { "features": ["pushable_location_stats:temperature"], "entities": {"location_id": location_ids}, From e593c1adb6bf58ca0e06e1cc0d149a21f55a052f Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:34:02 -0400 Subject: [PATCH 08/14] add pytest async to ci reqs Signed-off-by: Rob Howley --- sdk/python/requirements/py3.10-ci-requirements.txt | 3 +++ sdk/python/requirements/py3.11-ci-requirements.txt | 3 +++ sdk/python/requirements/py3.9-ci-requirements.txt | 3 +++ setup.py | 1 + 4 files changed, 10 insertions(+) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 0041e21703..7109a6feae 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -676,6 +676,7 @@ pyspark==3.5.2 pytest==7.4.4 # via # feast (setup.py) + # pytest-asyncio # pytest-benchmark # pytest-cov # pytest-env @@ -684,6 +685,8 @@ pytest==7.4.4 # pytest-ordering # pytest-timeout # pytest-xdist +pytest-asyncio==0.23.8 + # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 9e1225d9ed..eb8bbc280b 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -676,6 +676,7 @@ pyspark==3.5.2 pytest==7.4.4 # via # feast (setup.py) + # pytest-asyncio # pytest-benchmark # pytest-cov # pytest-env @@ -684,6 +685,8 @@ pytest==7.4.4 # pytest-ordering # pytest-timeout # pytest-xdist +pytest-asyncio==0.23.8 + # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index e7989843a8..2a380d6fc3 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -677,6 +677,7 @@ pyspark==3.5.2 pytest==7.4.4 # via # feast (setup.py) + # pytest-asyncio # pytest-benchmark # pytest-cov # pytest-env @@ -685,6 +686,8 @@ pytest==7.4.4 # pytest-ordering # pytest-timeout # pytest-xdist +pytest-asyncio==0.23.8 + # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) pytest-cov==5.0.0 diff --git a/setup.py b/setup.py index 7c75625d30..5ab57c906c 100644 --- a/setup.py +++ b/setup.py @@ -165,6 +165,7 @@ "psutil==5.9.0", "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 "pytest>=6.0.0,<8", + "pytest-asyncio<=0.24.0", "pytest-cov", "pytest-xdist", "pytest-benchmark>=3.4.1,<4", From f50c6cb68101b1937120ed470ba764a849d82f5c Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:52:24 -0400 Subject: [PATCH 09/14] be safe w cleanup in test fixture Signed-off-by: Rob Howley --- .../online_store/test_python_feature_server.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index f86d87ff2d..89f881f37d 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -63,13 +63,13 @@ def test_get_online_features(python_fs_client): async def test_push(python_fs_client): initial_temp = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] - )[0] + ) json_data = json.dumps( { "push_source_name": "location_stats_push_source", "df": { "location_id": [1], - "temperature": [initial_temp * 100], + "temperature": [initial_temp[0] * 100], "event_timestamp": [str(_utc_now())], "created": [str(_utc_now())], }, @@ -82,9 +82,10 @@ async def test_push(python_fs_client): # Check new pushed temperature is fetched assert response.status_code == 200 - assert await _get_temperatures_from_feature_server( + actual = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] - ) == [initial_temp * 100] + ) + assert actual == [initial_temp[0] * 100] @pytest.mark.integration @@ -142,5 +143,7 @@ def python_fs_client(environment, universal_data_sources, request): loop = asyncio.get_event_loop() loop.run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) - yield client - loop.run_until_complete(fs.close()) + try: + yield client + finally: + loop.run_until_complete(fs.close()) From 5c4a88a463fc355914d7a1390128461c92ba9bff Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 20:53:57 -0400 Subject: [PATCH 10/14] be safe w cleanup in test fixture Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 89f881f37d..082271ccdb 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -146,4 +146,4 @@ def python_fs_client(environment, universal_data_sources, request): try: yield client finally: - loop.run_until_complete(fs.close()) + asyncio.get_event_loop().run_until_complete(fs.close()) From 92dc4bde3b1959aae004bdde9cd0e185c34cc487 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 21:20:29 -0400 Subject: [PATCH 11/14] update pytest ini Signed-off-by: Rob Howley --- sdk/python/pytest.ini | 4 ++++ .../online_store/test_python_feature_server.py | 8 ++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index d87e4c07cb..a073676760 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -1,4 +1,6 @@ [pytest] +asyncio_mode = auto + markers = universal_offline_stores: mark a test as using all offline stores. universal_online_stores: mark a test as using all online stores. @@ -7,6 +9,8 @@ env = IS_TEST=True filterwarnings = + error::_pytest.warning_types.PytestConfigWarning + error::_pytest.warning_types.PytestUnhandledCoroutineWarning ignore::DeprecationWarning:pyspark.sql.pandas.*: ignore::DeprecationWarning:pyspark.sql.connect.*: ignore::DeprecationWarning:httpx.*: diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 082271ccdb..ee1d27d7e8 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -21,7 +21,7 @@ @pytest.mark.integration @pytest.mark.universal_online_stores -def test_get_online_features(python_fs_client): +async def test_get_online_features(python_fs_client): request_data_dict = { "features": [ "driver_stats:conv_rate", @@ -59,7 +59,6 @@ def test_get_online_features(python_fs_client): @pytest.mark.integration @pytest.mark.universal_online_stores -@pytest.mark.asyncio async def test_push(python_fs_client): initial_temp = await _get_temperatures_from_feature_server( python_fs_client, location_ids=[1] @@ -91,9 +90,6 @@ async def test_push(python_fs_client): @pytest.mark.integration @pytest.mark.universal_online_stores def test_push_source_does_not_exist(python_fs_client): - initial_temp = _get_temperatures_from_feature_server( - python_fs_client, location_ids=[1] - )[0] with pytest.raises( PushSourceNotFoundException, match="Unable to find push source 'push_source_does_not_exist'", @@ -105,7 +101,7 @@ def test_push_source_does_not_exist(python_fs_client): "push_source_name": "push_source_does_not_exist", "df": { "location_id": [1], - "temperature": [initial_temp * 100], + "temperature": [100], "event_timestamp": [str(_utc_now())], "created": [str(_utc_now())], }, From ee2249a6546b3047ca7625bce8d3daf7f37f48fe Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 22:05:50 -0400 Subject: [PATCH 12/14] not in a finally Signed-off-by: Rob Howley --- .../online_store/test_python_feature_server.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index ee1d27d7e8..5de8a6c7b5 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -136,10 +136,7 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) - loop = asyncio.get_event_loop() - loop.run_until_complete(fs.initialize()) + asyncio.get_event_loop().run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) - try: - yield client - finally: - asyncio.get_event_loop().run_until_complete(fs.close()) + yield client + asyncio.get_event_loop().run_until_complete(fs.close()) From 5c69273b0f7d2ba9c5d9d5986f3d8585bd028c8d Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 23:06:01 -0400 Subject: [PATCH 13/14] remove close Signed-off-by: Rob Howley --- .../tests/integration/online_store/test_python_feature_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 5de8a6c7b5..411e7a6f5b 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -139,4 +139,3 @@ def python_fs_client(environment, universal_data_sources, request): asyncio.get_event_loop().run_until_complete(fs.initialize()) client = TestClient(get_app(fs)) yield client - asyncio.get_event_loop().run_until_complete(fs.close()) From 385e3eabb16fe92caaae92f72f18160435470f96 Mon Sep 17 00:00:00 2001 From: Rob Howley Date: Wed, 23 Oct 2024 23:37:10 -0400 Subject: [PATCH 14/14] test client is a lifespan aware context manager Signed-off-by: Rob Howley --- .../integration/online_store/test_python_feature_server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sdk/python/tests/integration/online_store/test_python_feature_server.py b/sdk/python/tests/integration/online_store/test_python_feature_server.py index 411e7a6f5b..8e69f719bc 100644 --- a/sdk/python/tests/integration/online_store/test_python_feature_server.py +++ b/sdk/python/tests/integration/online_store/test_python_feature_server.py @@ -1,4 +1,3 @@ -import asyncio import json from typing import List @@ -136,6 +135,5 @@ def python_fs_client(environment, universal_data_sources, request): feast_objects.extend([driver(), customer(), location()]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) - asyncio.get_event_loop().run_until_complete(fs.initialize()) - client = TestClient(get_app(fs)) - yield client + with TestClient(get_app(fs)) as client: + yield client