Skip to content

Commit

Permalink
feat: Add decimal datatype to spark mapping to feast data type
Browse files Browse the repository at this point in the history
Signed-off-by: tanlocnguyen <tanlocnguyen296@gmail.com>
  • Loading branch information
ElliotNguyen68 committed Apr 8, 2024
1 parent 21e5434 commit d4637c5
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion sdk/python/feast/type_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -752,7 +752,7 @@ def _non_empty_value(value: Any) -> bool:

def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
# TODO not all spark types are convertible
# Current non-convertible types: interval, map, struct, structfield, decimal, binary
# Current non-convertible types: interval, map, struct, structfield, binary
type_map: Dict[str, ValueType] = {
"null": ValueType.UNKNOWN,
"byte": ValueType.BYTES,
Expand All @@ -762,6 +762,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
"bigint": ValueType.INT64,
"long": ValueType.INT64,
"double": ValueType.DOUBLE,
"decimal": ValueType.DOUBLE,
"float": ValueType.FLOAT,
"boolean": ValueType.BOOL,
"timestamp": ValueType.UNIX_TIMESTAMP,
Expand All @@ -774,6 +775,9 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
"array<boolean>": ValueType.BOOL_LIST,
"array<timestamp>": ValueType.UNIX_TIMESTAMP_LIST,
}
if spark_type_as_str.startswith('decimal'):
spark_type_as_str = "decimal"

# TODO: Find better way of doing this.
if not isinstance(spark_type_as_str, str) or spark_type_as_str not in type_map:
return ValueType.NULL
Expand Down

0 comments on commit d4637c5

Please sign in to comment.