From 1b8d849156c50a6350ec3200ac20f8b8ed26c2e8 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni Date: Wed, 12 Jun 2024 15:46:39 -0400 Subject: [PATCH] Draft changes to add remote online store to feast. Signed-off-by: Lokesh Rangineni Adding the integration test and remote online creator class so that it will fit into existing integration testing framework. Signed-off-by: Lokesh Rangineni Fix after rebase Signed-off-by: Lokesh Rangineni Removing the RemoteOnlineStoreCreator and adding custom integration test case. Incorporating the code review comments. Signed-off-by: Lokesh Rangineni reformatting the code, removing unnecessary braces. Signed-off-by: Lokesh Rangineni Trying to fix the errors reported in make lint-python Signed-off-by: Lokesh Rangineni Ran the command make format-python and trying to see if it fixes the lint errors. Signed-off-by: Lokesh Rangineni increasing the server start timeout to see if it fixes the integration test cases. Signed-off-by: Lokesh Rangineni checking changes after make format-python Signed-off-by: Lokesh Rangineni trying to see if this fixes the PR integrationt test failure. Signed-off-by: Lokesh Rangineni Signed-off-by: Lokesh Rangineni checking in the changes for make format-python Signed-off-by: Lokesh Rangineni Upgrading python version to 3.11, adding support for 3.11 as well. Signed-off-by: Lokesh Rangineni chore: Bump macOS runners to macos-13 (#4152) bump macos runner to 13 Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni chore: Use pixi to lock python dependencies in a single command (#4114) use pixi to lock python dependencies in a single command Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni feat: List all feature views (#4256) * feature: Adding type to base feature view Signed-off-by: Francisco Javier Arceo * fixed linter Signed-off-by: Francisco Javier Arceo * fixed type and meta Signed-off-by: Francisco Javier Arceo * adding new listing Signed-off-by: Francisco Javier Arceo * updated Signed-off-by: Francisco Javier Arceo * cleaning up changes Signed-off-by: Francisco Javier Arceo * reverting FV proto Signed-off-by: Francisco Javier Arceo * doing simple way Signed-off-by: Francisco Javier Arceo * added a test Signed-off-by: Francisco Javier Arceo * updated to add warnings Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo feat: Adding vector search for sqlite (#4176) * feat: Adding vector search for sqlite Signed-off-by: Francisco Javier Arceo * adding the sqlite_vss dependency Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo * latest progress Signed-off-by: Francisco Javier Arceo * uploading latest progress Signed-off-by: Francisco Javier Arceo * updated function Signed-off-by: Francisco Javier Arceo * adding configuration Signed-off-by: Francisco Javier Arceo * adding current progress Signed-off-by: Francisco Javier Arceo * updating requirements files Signed-off-by: Francisco Javier Arceo * moving to sqlite-vec Signed-off-by: Francisco Javier Arceo * updating sqlite.py Signed-off-by: Francisco Javier Arceo * checking in progress Signed-off-by: Francisco Javier Arceo * updated test type Signed-off-by: Francisco Javier Arceo * got the initialization working, nice Signed-off-by: Francisco Javier Arceo * checking in progress from last night Signed-off-by: Francisco Javier Arceo * removing unnecessary stuff Signed-off-by: Francisco Javier Arceo * fixing merge conflicts Signed-off-by: Francisco Javier Arceo * removing files changed accidentally] Signed-off-by: Francisco Javier Arceo * uploading current progress...things run but need to update the virtual table insertion Signed-off-by: Francisco Javier Arceo * linted Signed-off-by: Francisco Javier Arceo * adding working notes Signed-off-by: Francisco Javier Arceo * found a bug, original feature_store.py was only grabbing first feature view, adjusted Signed-off-by: Francisco Javier Arceo * cant use a string have to verify it is a proper FeatureView object Signed-off-by: Francisco Javier Arceo * updated got it working, need to fix some other stuff still Signed-off-by: Francisco Javier Arceo * working Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo * fixing some type issues Signed-off-by: Francisco Javier Arceo * fixed typing and lint issues Signed-off-by: Francisco Javier Arceo * updated dependencies Signed-off-by: Francisco Javier Arceo * fix for pixi and updating requirements Signed-off-by: Francisco Javier Arceo * fixed type Signed-off-by: Francisco Javier Arceo * linter Signed-off-by: Francisco Javier Arceo * testing sqlite_vec import Signed-off-by: Francisco Javier Arceo * adding minimal example test Signed-off-by: Francisco Javier Arceo * lint Signed-off-by: Francisco Javier Arceo * testing raw sqlite Signed-off-by: Francisco Javier Arceo * Printing package version * printing version Signed-off-by: Francisco Javier Arceo * updated requirements * rebuilding requirments Signed-off-by: Francisco Javier Arceo * only going to run this on 3.10 for now Signed-off-by: Francisco Javier Arceo * updated docs for sqlite caveats Signed-off-by: Francisco Javier Arceo * adding reason Signed-off-by: Francisco Javier Arceo * skipping Signed-off-by: Francisco Javier Arceo * updated tests Signed-off-by: Francisco Javier Arceo * removing print Signed-off-by: Francisco Javier Arceo * added method call Signed-off-by: Francisco Javier Arceo * added prubt Signed-off-by: Francisco Javier Arceo * added print Signed-off-by: Francisco Javier Arceo * removing print Signed-off-by: Francisco Javier Arceo * adding check in sqlite Signed-off-by: Francisco Javier Arceo * missed an = Signed-off-by: Francisco Javier Arceo * still running on 3.11 Signed-off-by: Francisco Javier Arceo * typo Signed-off-by: Francisco Javier Arceo * fix Signed-off-by: Francisco Javier Arceo * fix Signed-off-by: Francisco Javier Arceo * updated setup and docs Signed-off-by: Francisco Javier Arceo * renamed things Signed-off-by: Francisco Javier Arceo --------- Signed-off-by: Francisco Javier Arceo squashing the last 15 commits to one. Merge branch 'master' into feature/adding-remote-onlinestore-rebase Adding documentation and incorporating code review comment. Signed-off-by: Lokesh Rangineni Adding documentation and incorporating code review comment. Signed-off-by: Lokesh Rangineni Merge remote-tracking branch 'fork/feature/adding-remote-onlinestore-rebase' into feature/adding-remote-onlinestore-rebase Signed-off-by: Lokesh Rangineni --- .github/workflows/pr_integration_tests.yml | 2 +- Makefile | 6 ------ infra/scripts/pixi/pixi.toml | 8 ++++++-- sdk/python/feast/infra/online_stores/remote.py | 2 +- sdk/python/tests/conftest.py | 7 +++++-- setup.py | 1 - 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index bcf1be7d49..f4a9132d29 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/Makefile b/Makefile index d85f7c0e8a..669f6f3c69 100644 --- a/Makefile +++ b/Makefile @@ -75,12 +75,6 @@ lock-python-dependencies-all: pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt" pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt" -lock-python-dependencies-all: - pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.9-requirements.txt" - pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt" - pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.10-requirements.txt" - pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt" - benchmark-python: IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index 9a64336570..10179339f7 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -6,7 +6,7 @@ platforms = ["linux-64", "osx-arm64"] [tasks] [dependencies] -pip-tools = ">=7.4.1,<7.5" +uv = ">=0.1.39,<0.2" [feature.py39.dependencies] python = "~=3.9.0" @@ -14,6 +14,10 @@ python = "~=3.9.0" [feature.py310.dependencies] python = "~=3.10.0" +[feature.py311.dependencies] +python = "~=3.11.0" + [environments] py39 = ["py39"] -py310 = ["py310"] \ No newline at end of file +py310 = ["py310"] +py311 = ["py311"] diff --git a/sdk/python/feast/infra/online_stores/remote.py b/sdk/python/feast/infra/online_stores/remote.py index ca8b17e04d..19e1b7d515 100644 --- a/sdk/python/feast/infra/online_stores/remote.py +++ b/sdk/python/feast/infra/online_stores/remote.py @@ -119,7 +119,7 @@ def _construct_online_read_api_json_request( entity_keys: List[EntityKeyProto], table: FeatureView, requested_features: Optional[List[str]] = None, - ) -> json: + ) -> str: api_requested_features = [] if requested_features is not None: for requested_feature in requested_features: diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 48f482f542..d72e6911da 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -173,7 +173,7 @@ def simple_dataset_2() -> pd.DataFrame: def start_test_local_server(repo_path: str, port: int): fs = FeatureStore(repo_path) - fs.serve(host="localhost", port=port) + fs.serve("localhost", port, no_access_log=True) @pytest.fixture @@ -305,7 +305,10 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): @pytest.fixture def feature_server_endpoint(environment): - if not environment.python_feature_server or environment.provider != "local": + if ( + not environment.python_feature_server + or environment.test_repo_config.provider != "local" + ): yield environment.feature_store.get_feature_server_endpoint() return diff --git a/setup.py b/setup.py index 18ee91acf2..9b3d0e55e6 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,6 @@ "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", "dask[dataframe]>=2024.4.2", - "bowler", # Needed for automatic repo upgrades ] GCP_REQUIRED = [