diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 741d46e39e..c43b33c1d2 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -50,6 +50,7 @@ message OnDemandFeatureViewSpec { oneof transformation { UserDefinedFunction user_defined_function = 5; + OnDemandSubstraitTransformation on_demand_substrait_transformation = 9; } // Description of the on demand feature view. @@ -89,3 +90,7 @@ message UserDefinedFunction { // The string representation of the udf string body_text = 3; } + +message OnDemandSubstraitTransformation { + bytes substrait_plan = 1; +} \ No newline at end of file diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 706f2ec4e4..586286a3d4 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,5 +1,6 @@ import copy import functools +import inspect import warnings from datetime import datetime from types import FunctionType @@ -17,6 +18,7 @@ from feast.feature_view_projection import FeatureViewProjection from feast.field import Field, from_value_type from feast.on_demand_pandas_transformation import OnDemandPandasTransformation +from feast.on_demand_substrait_transformation import OnDemandSubstraitTransformation from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( OnDemandFeatureView as OnDemandFeatureViewProto, ) @@ -210,6 +212,9 @@ def to_proto(self) -> OnDemandFeatureViewProto: user_defined_function=self.transformation.to_proto() if type(self.transformation) == OnDemandPandasTransformation else None, + on_demand_substrait_transformation=self.transformation.to_proto() # type: ignore + if type(self.transformation) == OnDemandSubstraitTransformation + else None, description=self.description, tags=self.tags, owner=self.owner, @@ -255,6 +260,13 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): transformation = OnDemandPandasTransformation.from_proto( on_demand_feature_view_proto.spec.user_defined_function ) + elif ( + on_demand_feature_view_proto.spec.WhichOneof("transformation") + == "on_demand_substrait_transformation" + ): + transformation = OnDemandSubstraitTransformation.from_proto( + on_demand_feature_view_proto.spec.on_demand_substrait_transformation + ) else: raise Exception("At least one transformation type needs to be provided") @@ -460,10 +472,47 @@ def mainify(obj) -> None: obj.__module__ = "__main__" def decorator(user_function): - udf_string = dill.source.getsource(user_function) - mainify(user_function) + return_annotation = inspect.signature(user_function).return_annotation + if ( + return_annotation + and return_annotation.__module__ == "ibis.expr.types.relations" + and return_annotation.__name__ == "Table" + ): + import ibis + import ibis.expr.datatypes as dt + from ibis_substrait.compiler.core import SubstraitCompiler + + compiler = SubstraitCompiler() + + input_fields: Field = [] + + for s in sources: + if type(s) == FeatureView: + fields = s.projection.features + else: + fields = s.features + + input_fields.extend( + [ + ( + f.name, + dt.dtype( + feast_value_type_to_pandas_type(f.dtype.to_value_type()) + ), + ) + for f in fields + ] + ) + + expr = user_function(ibis.table(input_fields, "t")) - transformation = OnDemandPandasTransformation(user_function, udf_string) + transformation = OnDemandSubstraitTransformation( + substrait_plan=compiler.compile(expr).SerializeToString() + ) + else: + udf_string = dill.source.getsource(user_function) + mainify(user_function) + transformation = OnDemandPandasTransformation(user_function, udf_string) on_demand_feature_view_obj = OnDemandFeatureView( name=user_function.__name__, diff --git a/sdk/python/feast/on_demand_substrait_transformation.py b/sdk/python/feast/on_demand_substrait_transformation.py new file mode 100644 index 0000000000..4e92e77dc8 --- /dev/null +++ b/sdk/python/feast/on_demand_substrait_transformation.py @@ -0,0 +1,50 @@ +import pandas as pd +import pyarrow +import pyarrow.substrait as substrait # type: ignore # noqa + +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + OnDemandSubstraitTransformation as OnDemandSubstraitTransformationProto, +) + + +class OnDemandSubstraitTransformation: + def __init__(self, substrait_plan: bytes): + """ + Creates an OnDemandSubstraitTransformation object. + + Args: + substrait_plan: The user-provided substrait plan. + """ + self.substrait_plan = substrait_plan + + def transform(self, df: pd.DataFrame) -> pd.DataFrame: + def table_provider(names, schema: pyarrow.Schema): + return pyarrow.Table.from_pandas(df[schema.names]) + + table: pyarrow.Table = pyarrow.substrait.run_query( + self.substrait_plan, table_provider=table_provider + ).read_all() + return table.to_pandas() + + def __eq__(self, other): + if not isinstance(other, OnDemandSubstraitTransformation): + raise TypeError( + "Comparisons should only involve OnDemandSubstraitTransformation class objects." + ) + + if not super().__eq__(other): + return False + + return self.substrait_plan == other.substrait_plan + + def to_proto(self) -> OnDemandSubstraitTransformationProto: + return OnDemandSubstraitTransformationProto(substrait_plan=self.substrait_plan) + + @classmethod + def from_proto( + cls, + on_demand_substrait_transformation_proto: OnDemandSubstraitTransformationProto, + ): + return OnDemandSubstraitTransformation( + substrait_plan=on_demand_substrait_transformation_proto.substrait_plan + ) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index f20bc05df9..45c4eb765d 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -10,7 +10,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -34,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==4.0 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -55,15 +57,17 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -122,7 +126,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -137,7 +141,7 @@ cryptography==41.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -213,9 +217,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -226,10 +230,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -244,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -264,13 +264,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -282,15 +282,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -304,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -312,11 +312,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -337,7 +343,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.2 # via jupyterlab -ipython==8.21.0 +ipython==8.22.1 # via # great-expectations # ipykernel @@ -367,7 +373,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -412,7 +418,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -428,6 +434,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -441,6 +449,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -455,7 +465,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -463,6 +473,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -477,7 +489,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -491,7 +503,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -501,6 +513,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -519,6 +532,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -536,6 +550,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -544,6 +559,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -552,7 +569,7 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -580,7 +597,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -598,6 +615,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -621,7 +639,10 @@ pyarrow==15.0.0 # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework pyasn1==0.5.1 # via # pyasn1-modules @@ -634,12 +655,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -648,6 +669,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -659,14 +681,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -699,6 +723,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -712,6 +737,7 @@ python-json-logger==2.0.7 pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -719,6 +745,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -769,7 +796,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -805,7 +834,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -826,13 +855,19 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -865,6 +900,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -893,7 +929,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -913,11 +949,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -931,11 +967,13 @@ typing-extensions==4.9.0 # azure-storage-blob # fastapi # great-expectations + # ibis-framework # mypy # pydantic # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 3943662d01..7141cd0f25 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -38,7 +38,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -131,7 +131,7 @@ partd==1.4.1 # via dask proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -141,11 +141,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -212,13 +212,12 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # sqlalchemy2-stubs + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index afa43ec2a2..b006c1c621 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,14 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # - alabaster==0.7.13 # via sphinx altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -35,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==3.1.2 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -66,11 +67,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -129,7 +130,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -219,9 +220,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -232,10 +233,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -250,7 +247,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -270,13 +267,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -288,15 +285,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -310,7 +307,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -318,11 +315,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==4.1.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==2.29.1 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -344,6 +347,7 @@ importlib-metadata==6.11.0 # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -384,7 +388,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -429,7 +433,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -445,6 +449,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -458,6 +464,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -472,7 +480,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -480,6 +488,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==0.6.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -494,7 +504,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -508,7 +518,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -518,6 +528,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -536,6 +547,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -553,6 +565,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -561,6 +574,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -571,7 +586,7 @@ pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) pkgutil-resolve-name==1.3.10 # via jsonschema @@ -601,7 +616,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -617,6 +632,7 @@ protobuf==4.23.3 # grpcio-status # grpcio-testing # grpcio-tools + # ibis-substrait # mypy-protobuf # proto-plus psutil==5.9.0 @@ -655,12 +671,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -669,6 +685,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -680,14 +697,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -720,6 +739,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -734,6 +754,7 @@ pytz==2024.1 # via # babel # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -741,6 +762,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -791,7 +813,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -819,6 +843,7 @@ six==1.16.0 # isodate # kubernetes # mock + # multipledispatch # pandavro # python-dateutil # rfc3339-validator @@ -829,7 +854,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -850,9 +875,13 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==10.6.4 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 @@ -889,6 +918,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -917,7 +947,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -937,11 +967,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -949,6 +979,7 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.9.0 # via + # annotated-types # anyio # async-lru # azure-core @@ -956,13 +987,16 @@ typing-extensions==4.9.0 # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic # pydantic-core + # rich # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 079064a9ec..541beecf0d 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -89,6 +89,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -136,7 +137,7 @@ pkgutil-resolve-name==1.3.10 # via jsonschema proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -146,11 +147,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -184,7 +185,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -209,6 +212,7 @@ types-protobuf==4.24.0.20240129 # via mypy-protobuf typing-extensions==4.9.0 # via + # annotated-types # anyio # fastapi # mypy @@ -216,13 +220,12 @@ typing-extensions==4.9.0 # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 6c26f889e2..e17d545d38 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -10,7 +10,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -34,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==4.0 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -55,15 +57,17 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -122,7 +126,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -137,7 +141,7 @@ cryptography==41.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -213,9 +217,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -226,10 +230,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -244,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -264,13 +264,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -282,15 +282,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -304,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -312,11 +312,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -338,6 +344,7 @@ importlib-metadata==6.11.0 # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 @@ -374,7 +381,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -419,7 +426,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -435,6 +442,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -448,6 +457,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -462,7 +473,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -470,6 +481,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -484,7 +497,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -498,7 +511,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -508,6 +521,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -526,6 +540,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -543,6 +558,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -551,6 +567,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -559,7 +577,7 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -587,7 +605,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -605,6 +623,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -628,7 +647,10 @@ pyarrow==15.0.0 # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework pyasn1==0.5.1 # via # pyasn1-modules @@ -641,12 +663,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -655,6 +677,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -666,14 +689,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -706,6 +731,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -719,6 +745,7 @@ python-json-logger==2.0.7 pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -726,6 +753,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -776,7 +804,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -814,7 +844,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -835,13 +865,19 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -874,6 +910,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -902,7 +939,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -922,11 +959,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -941,6 +978,7 @@ typing-extensions==4.9.0 # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic @@ -948,6 +986,7 @@ typing-extensions==4.9.0 # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 182cb7ad07..12c14a03ae 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -38,7 +38,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -89,6 +89,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 @@ -131,7 +132,7 @@ partd==1.4.1 # via dask proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -141,11 +142,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -179,7 +180,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -211,8 +214,9 @@ typing-extensions==4.9.0 # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 # via feast (setup.py) diff --git a/sdk/python/tests/unit/test_on_demand_substrait_transformation.py b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py new file mode 100644 index 0000000000..c9d30c5b7a --- /dev/null +++ b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py @@ -0,0 +1,112 @@ +import os +import tempfile +from datetime import datetime, timedelta + +import pandas as pd + +from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast.driver_test_data import create_driver_hourly_stats_df +from feast.field import Field +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + + +def test_ibis_pandas_parity(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_substrait_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + @on_demand_feature_view( + sources=[driver_stats_fv], + schema=[Field(name="conv_rate_plus_acc", dtype=Float64)], + ) + def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_acc"] = inputs["conv_rate"] + inputs["acc_rate"] + return df + + from ibis.expr.types import Table + + @on_demand_feature_view( + sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], + schema=[Field(name="conv_rate_plus_acc_substrait", dtype=Float64)], + ) + def substrait_view(inputs: Table) -> Table: + return inputs.select( + (inputs["conv_rate"] + inputs["acc_rate"]).name( + "conv_rate_plus_acc_substrait" + ) + ) + + store.apply( + [driver, driver_stats_source, driver_stats_fv, substrait_view, pandas_view] + ) + + entity_df = pd.DataFrame.from_dict( + { + # entity's join key -> entity values + "driver_id": [1001, 1002, 1003], + # "event_timestamp" (reserved key) -> timestamps + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + } + ) + + training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "substrait_view:conv_rate_plus_acc_substrait", + "pandas_view:conv_rate_plus_acc", + ], + ).to_df() + + assert training_df["conv_rate_plus_acc"].equals( + training_df["conv_rate_plus_acc_substrait"] + ) diff --git a/setup.py b/setup.py index c14d64557a..3aaaa51ea0 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,11 @@ "hazelcast-python-client>=5.1", ] +IBIS_REQUIRED = [ + "ibis-framework", + "ibis-substrait" +] + CI_REQUIRED = ( [ "build", @@ -201,6 +206,7 @@ + AZURE_REQUIRED + ROCKSET_REQUIRED + HAZELCAST_REQUIRED + + IBIS_REQUIRED ) @@ -368,6 +374,7 @@ def run(self): "cassandra": CASSANDRA_REQUIRED, "hazelcast": HAZELCAST_REQUIRED, "rockset": ROCKSET_REQUIRED, + "ibis": IBIS_REQUIRED }, include_package_data=True, license="Apache",