Merge branch 'master' into ibis-point-in-time-join

.github/workflows/pr_integration_tests.yml

@@ -167,4 +167,4 @@ jobs:
           SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
           SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
           SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
+        run: make test-python-integration

.github/workflows/pr_local_integration_tests.yml

@@ -61,9 +61,4 @@ jobs:
         run: make install-python-ci-dependencies
       - name: Test local integration tests
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
-        env:
-          FEAST_USAGE: "False"
-          IS_TEST: "True"
-          FEAST_LOCAL_ONLINE_CONTAINER: "True"
-          FEAST_IS_LOCAL_TEST: "True"
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes --integration -k "not gcs_registry and not s3_registry and not test_lambda_materialization and not test_snowflake_materialization" sdk/python/tests
+        run: make test-python-integration-local

.github/workflows/unit_tests.yml

@@ -23,17 +23,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
-      - name: Install mysql on macOS
-        if:  startsWith(matrix.os, 'macOS')
-        run: |
-          brew install mysql
-          PATH=$PATH:/usr/local/mysql/bin
-      - name: Work around Homebrew MySQL being broken
-        # See https://github.com/Homebrew/homebrew-core/issues/130258 for more details.
-        if:  startsWith(matrix.os, 'macOS')
-        run: |
-          brew install zlib
-          ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -56,7 +45,7 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
-        run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests
+        run: make test-python-unit
 
 
   unit-test-ui:

Makefile

@@ -28,7 +28,7 @@ format: format-python format-java
 
 lint: lint-python lint-java
 
-test: test-python test-java
+test: test-python-unit test-java
 
 protos: compile-protos-python compile-protos-docs
 
@@ -63,32 +63,26 @@ benchmark-python:
 benchmark-python-local:
 	FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
 
-test-python:
-	FEAST_USAGE=False \
-	IS_TEST=True \
-	python -m pytest -n 8 sdk/python/tests \
+test-python-unit:
+	python -m pytest -n 8 --color=yes sdk/python/tests
 
 test-python-integration:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	python -m pytest -n 8 --integration --color=yes --durations=5 --timeout=1200 --timeout_method=thread sdk/python/tests
 
 test-python-integration-local:
 	@(docker info > /dev/null 2>&1 && \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		FEAST_IS_LOCAL_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
-		python -m pytest -n 8 --integration \
+		python -m pytest -n 8 --color=yes --integration \
 			-k "not gcs_registry and \
  				not s3_registry and \
  				not test_lambda_materialization and \
- 				not test_snowflake" \
+ 				not test_snowflake_materialization" \
 		sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
 
 test-python-integration-container:
 	@(docker info > /dev/null 2>&1 && \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		FEAST_LOCAL_ONLINE_CONTAINER=True \
 		python -m pytest -n 8 --integration sdk/python/tests \
 	) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
@@ -97,7 +91,6 @@ test-python-universal-spark:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.spark_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.spark_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_historical_retrieval_fails_on_validation and \
 			not test_historical_retrieval_with_validation and \
@@ -121,7 +114,6 @@ test-python-universal-trino:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.trino_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.trino_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_historical_retrieval_fails_on_validation and \
 			not test_historical_retrieval_with_validation and \
@@ -148,7 +140,6 @@ test-python-universal-mssql:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.mssql_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.mssql_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
 	FEAST_LOCAL_ONLINE_CONTAINER=True \
  	python -m pytest -n 8 --integration \
  	 	-k "not gcs_registry and \
@@ -166,7 +157,6 @@ test-python-universal-athena:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.athena_repo_configuration \
 	PYTEST_PLUGINS=feast.infra.offline_stores.contrib.athena_offline_store.tests \
- 	FEAST_USAGE=False IS_TEST=True \
 	ATHENA_REGION=ap-northeast-2 \
 	ATHENA_DATA_SOURCE=AwsDataCatalog \
 	ATHENA_DATABASE=default \
@@ -190,7 +180,6 @@ test-python-universal-athena:
 test-python-universal-duckdb:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \
-	FEAST_USAGE=False IS_TEST=True \
 	python -m pytest -n 8 --integration \
 		-k "not test_nullable_online_store and \
 			not gcs_registry and \
@@ -204,8 +193,6 @@ test-python-universal-postgres-offline:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_historical_retrieval_with_validation and \
 				not test_historical_features_persisting and \
@@ -226,8 +213,6 @@ test-python-universal-postgres-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.postgres_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.feast.infra.offline_stores.contrib.postgres_offline_store.tests \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -247,8 +232,6 @@ test-python-universal-postgres-online:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.mysql_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -268,17 +251,13 @@ test-python-universal-cassandra:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
-	FEAST_USAGE=False \
-	IS_TEST=True \
 	python -m pytest -x --integration \
 	sdk/python/tests
 
 test-python-universal-hazelcast:
 	PYTHONPATH='.' \
 		FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.hazelcast_repo_configuration \
 		PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.hazelcast \
-		FEAST_USAGE=False \
-		IS_TEST=True \
 		python -m pytest -n 8 --integration \
  			-k "not test_universal_cli and \
  				not test_go_feature_server and \
@@ -298,8 +277,6 @@ test-python-universal-cassandra-no-cloud-providers:
 	PYTHONPATH='.' \
 	FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.cassandra_repo_configuration \
 	PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \
-	FEAST_USAGE=False \
-	IS_TEST=True \
 	python -m pytest -x --integration \
 	-k "not test_lambda_materialization_consistency   and \
 	  not test_apply_entity_integration               and \
@@ -314,7 +291,7 @@ test-python-universal-cassandra-no-cloud-providers:
 	sdk/python/tests
 
 test-python-universal:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	python -m pytest -n 8 --integration sdk/python/tests
 
 format-python:
 	# Sort

docs/reference/offline-stores/spark.md

@@ -30,6 +30,8 @@ offline_store:
         spark.sql.catalogImplementation: "hive"
         spark.sql.parser.quotedRegexColumnNames: "true"
         spark.sql.session.timeZone: "UTC"
+        spark.sql.execution.arrow.fallback.enabled: "true"
+        spark.sql.execution.arrow.pyspark.enabled: "true"
 online_store:
     path: data/online_store.db
 ```

protos/feast/core/OnDemandFeatureView.proto

@@ -27,6 +27,7 @@ import "feast/core/FeatureView.proto";
 import "feast/core/FeatureViewProjection.proto";
 import "feast/core/Feature.proto";
 import "feast/core/DataSource.proto";
+import "feast/core/Transformation.proto";
 
 message OnDemandFeatureView {
     // User-specified specifications of this feature view.
@@ -48,10 +49,10 @@ message OnDemandFeatureViewSpec {
     // Map of sources for this feature view.
     map<string, OnDemandSource> sources = 4;
 
-    oneof transformation {
-        UserDefinedFunction user_defined_function = 5;
-        OnDemandSubstraitTransformation on_demand_substrait_transformation = 9;
-    }
+    UserDefinedFunction user_defined_function = 5 [deprecated = true];
+
+    // Oneof with {user_defined_function, on_demand_substrait_transformation}
+    FeatureTransformationV2 feature_transformation = 10;
 
     // Description of the on demand feature view.
     string description = 6;
@@ -61,6 +62,7 @@ message OnDemandFeatureViewSpec {
 
     // Owner of the on demand feature view.
     string owner = 8;
+    string mode = 11;
 }
 
 message OnDemandFeatureViewMeta {
@@ -81,6 +83,8 @@ message OnDemandSource {
 
 // Serialized representation of python function.
 message UserDefinedFunction {
+    option deprecated = true;
+
     // The function name
     string name = 1;
 
@@ -90,7 +94,3 @@ message UserDefinedFunction {
     // The string representation of the udf
     string body_text = 3;
 }
-
-message OnDemandSubstraitTransformation {
-    bytes substrait_plan = 1;
-}

protos/feast/core/Registry.proto

@@ -27,7 +27,6 @@ import "feast/core/FeatureTable.proto";
 import "feast/core/FeatureView.proto";
 import "feast/core/InfraObject.proto";
 import "feast/core/OnDemandFeatureView.proto";
-import "feast/core/RequestFeatureView.proto";
 import "feast/core/StreamFeatureView.proto";
 import "feast/core/DataSource.proto";
 import "feast/core/SavedDataset.proto";
@@ -41,7 +40,6 @@ message Registry {
     repeated FeatureView feature_views = 6;
     repeated DataSource data_sources = 12;
     repeated OnDemandFeatureView on_demand_feature_views = 8;
-    repeated RequestFeatureView request_feature_views = 9;
     repeated StreamFeatureView stream_feature_views = 14;
     repeated FeatureService feature_services = 7;
     repeated SavedDataset saved_datasets = 11;

protos/feast/core/StreamFeatureView.proto

@@ -29,6 +29,7 @@ import "feast/core/FeatureView.proto";
 import "feast/core/Feature.proto";
 import "feast/core/DataSource.proto";
 import "feast/core/Aggregation.proto";
+import "feast/core/Transformation.proto";
 
 message StreamFeatureView {
     // User-specified specifications of this feature view.
@@ -77,7 +78,8 @@ message StreamFeatureViewSpec {
     bool online = 12;
 
     // Serialized function that is encoded in the streamfeatureview
-    UserDefinedFunction user_defined_function = 13;
+    UserDefinedFunction user_defined_function = 13 [deprecated = true];
+
 
     // Mode of execution
     string mode = 14;
@@ -87,5 +89,8 @@ message StreamFeatureViewSpec {
 
     // Timestamp field for aggregation
     string timestamp_field = 16;
+
+    // Oneof with {user_defined_function, on_demand_substrait_transformation}
+    FeatureTransformationV2 feature_transformation = 17;
 }
 

protos/feast/core/Transformation.proto

@@ -0,0 +1,32 @@
+syntax = "proto3";
+package feast.core;
+
+option go_package = "github.com/feast-dev/feast/go/protos/feast/core";
+option java_outer_classname = "FeatureTransformationProto";
+option java_package = "feast.proto.core";
+
+import "google/protobuf/duration.proto";
+
+// Serialized representation of python function.
+message UserDefinedFunctionV2 {
+    // The function name
+    string name = 1;
+
+    // The python-syntax function body (serialized by dill)
+    bytes body = 2;
+
+    // The string representation of the udf
+    string body_text = 3;
+}
+
+// A feature transformation executed as a user-defined function
+message FeatureTransformationV2 {
+  oneof transformation {
+    UserDefinedFunctionV2 user_defined_function = 1;
+    SubstraitTransformationV2 substrait_transformation = 2;
+  }
+}
+
+message SubstraitTransformationV2 {
+  bytes substrait_plan = 1;
+}