feat: Enable Arrow-based columnar data transfers

Signed-off-by: tanlocnguyen <tanlocnguyen296@gmail.com>
feast-dev · Mar 19, 2024 · 2f37e07 · 2f37e07
1 parent 42a7b81
commit 2f37e07
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/docs/reference/offline-stores/spark.md b/docs/reference/offline-stores/spark.md
@@ -30,6 +30,8 @@ offline_store:
         spark.sql.catalogImplementation: "hive"
         spark.sql.parser.quotedRegexColumnNames: "true"
         spark.sql.session.timeZone: "UTC"
+        spark.sql.execution.arrow.fallback.enabled: "true"
+        spark.sql.execution.arrow.pyspark.enabled: "true"
 online_store:
     path: data/online_store.db
 ```

diff --git a/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml b/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml
@@ -12,6 +12,8 @@ offline_store:
     spark.sql.catalogImplementation: "hive"
     spark.sql.parser.quotedRegexColumnNames: "true"
     spark.sql.session.timeZone: "UTC"
+    spark.sql.execution.arrow.fallback.enabled: "true"
+    spark.sql.execution.arrow.pyspark.enabled: "true"
 online_store:
   path: data/online_store.db
 entity_key_serialization_version: 2