feat(pyspark): add DateDiff operation

ibis-project · Mar 27, 2023 · bfd6109 · bfd6109
1 parent af9a2c8
commit bfd6109
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 7 deletions.
diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py
@@ -1536,8 +1536,11 @@ def compile_date_sub(t, op, **kwargs):
 
 @compiles(ops.DateDiff)
 def compile_date_diff(t, op, **kwargs):
-    raise com.UnsupportedOperationError(
-        'PySpark backend does not support DateDiff as there is no timedelta type.'
+    left = t.translate(op.left, **kwargs)
+    right = t.translate(op.right, **kwargs)
+
+    return F.concat(F.lit("INTERVAL '"), F.datediff(left, right), F.lit("' DAY")).cast(
+        pt.DayTimeIntervalType(pt.DayTimeIntervalType.DAY, pt.DayTimeIntervalType.DAY)
     )
 
 

diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py
@@ -855,11 +855,6 @@ def convert_to_offset(x):
             lambda t, _: t.timestamp_col.dt.floor('d') - date_value,
             id='date-subtract-date',
             marks=[
-                pytest.mark.notimpl(
-                    ["pyspark"],
-                    raises=com.UnsupportedOperationError,
-                    reason="PySpark backend does not support DateDiff as there is no timedelta type.",
-                ),
                 pytest.mark.notimpl(["bigquery"], raises=com.OperationNotDefinedError),
                 pytest.mark.notimpl(
                     ["druid"],