-
Notifications
You must be signed in to change notification settings - Fork 998
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial scaffolding for on demand feature view (#1803)
* Initial scaffolding for on demand feature view, with initial support for transforms on online fetches Signed-off-by: Danny Chiao <danny@tecton.ai> * Fixing comments Signed-off-by: Danny Chiao <danny@tecton.ai> * Comments Signed-off-by: Danny Chiao <danny@tecton.ai> * Added basic test Signed-off-by: Danny Chiao <danny@tecton.ai> * Simplifying function serialization Signed-off-by: Danny Chiao <danny@tecton.ai> * Refactor logic into odfv Signed-off-by: Danny Chiao <danny@tecton.ai>
- Loading branch information
Showing
12 changed files
with
434 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // | ||
| // Copyright 2020 The Feast Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // https://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| // | ||
|
|
||
|
|
||
| syntax = "proto3"; | ||
| package feast.core; | ||
|
|
||
| option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; | ||
| option java_outer_classname = "OnDemandFeatureViewProto"; | ||
| option java_package = "feast.proto.core"; | ||
|
|
||
| import "feast/core/FeatureView.proto"; | ||
| import "feast/core/Feature.proto"; | ||
|
|
||
| message OnDemandFeatureView { | ||
| // User-specified specifications of this feature view. | ||
| OnDemandFeatureViewSpec spec = 1; | ||
| } | ||
|
|
||
| message OnDemandFeatureViewSpec { | ||
| // Name of the feature view. Must be unique. Not updated. | ||
| string name = 1; | ||
|
|
||
| // Name of Feast project that this feature view belongs to. | ||
| string project = 2; | ||
|
|
||
| // List of features specifications for each feature defined with this feature view. | ||
| repeated FeatureSpecV2 features = 3; | ||
|
|
||
| // List of features specifications for each feature defined with this feature view. | ||
| // TODO(adchia): add support for request data | ||
| map<string, FeatureView> inputs = 4; | ||
|
|
||
| UserDefinedFunction user_defined_function = 5; | ||
| } | ||
|
|
||
| // Serialized representation of python function. | ||
| message UserDefinedFunction { | ||
| // The function name | ||
| string name = 1; | ||
|
|
||
| // The python-syntax function body (serialized by dill) | ||
| bytes body = 2; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| import functools | ||
| from types import MethodType | ||
| from typing import Dict, List | ||
|
|
||
| import dill | ||
| import pandas as pd | ||
|
|
||
| from feast.feature import Feature | ||
| from feast.feature_view import FeatureView | ||
| from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( | ||
| OnDemandFeatureView as OnDemandFeatureViewProto, | ||
| ) | ||
| from feast.protos.feast.core.OnDemandFeatureView_pb2 import OnDemandFeatureViewSpec | ||
| from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( | ||
| UserDefinedFunction as UserDefinedFunctionProto, | ||
| ) | ||
| from feast.usage import log_exceptions | ||
| from feast.value_type import ValueType | ||
|
|
||
|
|
||
| class OnDemandFeatureView: | ||
| """ | ||
| An OnDemandFeatureView defines on demand transformations on existing feature view values and request data. | ||
| Args: | ||
| name: Name of the group of features. | ||
| features: Output schema of transformation with feature names | ||
| inputs: The input feature views passed into the transform. | ||
| udf: User defined transformation function that takes as input pandas dataframes | ||
| """ | ||
|
|
||
| name: str | ||
| features: List[Feature] | ||
| inputs: Dict[str, FeatureView] | ||
| udf: MethodType | ||
|
|
||
| @log_exceptions | ||
| def __init__( | ||
| self, | ||
| name: str, | ||
| features: List[Feature], | ||
| inputs: Dict[str, FeatureView], | ||
| udf: MethodType, | ||
| ): | ||
| """ | ||
| Creates an OnDemandFeatureView object. | ||
| """ | ||
|
|
||
| self.name = name | ||
| self.features = features | ||
| self.inputs = inputs | ||
| self.udf = udf | ||
|
|
||
| def to_proto(self) -> OnDemandFeatureViewProto: | ||
| """ | ||
| Converts an on demand feature view object to its protobuf representation. | ||
| Returns: | ||
| A OnDemandFeatureViewProto protobuf. | ||
| """ | ||
| spec = OnDemandFeatureViewSpec( | ||
| name=self.name, | ||
| features=[feature.to_proto() for feature in self.features], | ||
| inputs={k: fv.to_proto() for k, fv in self.inputs.items()}, | ||
| user_defined_function=UserDefinedFunctionProto( | ||
| name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), | ||
| ), | ||
| ) | ||
|
|
||
| return OnDemandFeatureViewProto(spec=spec) | ||
|
|
||
| @classmethod | ||
| def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): | ||
| """ | ||
| Creates an on demand feature view from a protobuf representation. | ||
| Args: | ||
| on_demand_feature_view_proto: A protobuf representation of an on-demand feature view. | ||
| Returns: | ||
| A OnDemandFeatureView object based on the on-demand feature view protobuf. | ||
| """ | ||
| on_demand_feature_view_obj = cls( | ||
| name=on_demand_feature_view_proto.spec.name, | ||
| features=[ | ||
| Feature( | ||
| name=feature.name, | ||
| dtype=ValueType(feature.value_type), | ||
| labels=dict(feature.labels), | ||
| ) | ||
| for feature in on_demand_feature_view_proto.spec.features | ||
| ], | ||
| inputs={ | ||
| feature_view_name: FeatureView.from_proto(feature_view_proto) | ||
| for feature_view_name, feature_view_proto in on_demand_feature_view_proto.spec.inputs.items() | ||
| }, | ||
| udf=dill.loads( | ||
| on_demand_feature_view_proto.spec.user_defined_function.body | ||
| ), | ||
| ) | ||
|
|
||
| return on_demand_feature_view_obj | ||
|
|
||
| def get_transformed_features_df( | ||
| self, full_feature_names: bool, df_with_features: pd.DataFrame | ||
| ) -> pd.DataFrame: | ||
| # Apply on demand transformations | ||
| # TODO(adchia): Include only the feature values from the specified input FVs in the ODFV. | ||
| # Copy over un-prefixed features even if not requested since transform may need it | ||
| columns_to_cleanup = [] | ||
| if full_feature_names: | ||
| for input_fv in self.inputs.values(): | ||
| for feature in input_fv.features: | ||
| full_feature_ref = f"{input_fv.name}__{feature.name}" | ||
| if full_feature_ref in df_with_features.keys(): | ||
| df_with_features[feature.name] = df_with_features[ | ||
| full_feature_ref | ||
| ] | ||
| columns_to_cleanup.append(feature.name) | ||
|
|
||
| # Compute transformed values and apply to each result row | ||
| df_with_transformed_features = self.udf.__call__(df_with_features) | ||
|
|
||
| # Cleanup extra columns used for transformation | ||
| df_with_features.drop(columns=columns_to_cleanup, inplace=True) | ||
| return df_with_transformed_features | ||
|
|
||
|
|
||
| def on_demand_feature_view(features: List[Feature], inputs: Dict[str, FeatureView]): | ||
| """ | ||
| Declare an on-demand feature view | ||
| :param features: Output schema with feature names | ||
| :param inputs: The inputs passed into the transform. | ||
| :return: An On Demand Feature View. | ||
| """ | ||
|
|
||
| def decorator(user_function): | ||
| on_demand_feature_view_obj = OnDemandFeatureView( | ||
| name=user_function.__name__, | ||
| inputs=inputs, | ||
| features=features, | ||
| udf=user_function, | ||
| ) | ||
| functools.update_wrapper( | ||
| wrapper=on_demand_feature_view_obj, wrapped=user_function | ||
| ) | ||
| return on_demand_feature_view_obj | ||
|
|
||
| return decorator |
Oops, something went wrong.