diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py index f705dd488429..c4dab04ac275 100644 --- a/ibis/backends/pyspark/compiler.py +++ b/ibis/backends/pyspark/compiler.py @@ -9,6 +9,7 @@ import pyspark import pyspark.sql.functions as F import pyspark.sql.types as pt +from packaging.version import parse as vparse from pyspark.sql import Window from pyspark.sql.functions import PandasUDFType, pandas_udf @@ -1557,14 +1558,20 @@ def compile_date_sub(t, op, **kwargs): ) -@compiles(ops.DateDiff) -def compile_date_diff(t, op, **kwargs): - left = t.translate(op.left, **kwargs) - right = t.translate(op.right, **kwargs) +if vparse(pyspark.__version__) >= vparse("3.3"): - return F.concat(F.lit("INTERVAL '"), F.datediff(left, right), F.lit("' DAY")).cast( - pt.DayTimeIntervalType(pt.DayTimeIntervalType.DAY, pt.DayTimeIntervalType.DAY) - ) + @compiles(ops.DateDiff) + def compile_date_diff(t, op, **kwargs): + left = t.translate(op.left, **kwargs) + right = t.translate(op.right, **kwargs) + + return F.concat( + F.lit("INTERVAL '"), F.datediff(left, right), F.lit("' DAY") + ).cast( + pt.DayTimeIntervalType( + pt.DayTimeIntervalType.DAY, pt.DayTimeIntervalType.DAY + ) + ) @compiles(ops.TimestampAdd) diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py index a4b6bbae8593..04544fa5aaab 100644 --- a/ibis/backends/tests/test_temporal.py +++ b/ibis/backends/tests/test_temporal.py @@ -949,6 +949,11 @@ def convert_to_offset(x): ), id="date-subtract-date", marks=[ + pytest.mark.xfail_version( + pyspark=["pyspark<3.3"], + raises=AttributeError, + reason="DayTimeIntervalType added in pyspark 3.3", + ), pytest.mark.notimpl(["bigquery"], raises=com.OperationNotDefinedError), pytest.mark.notimpl( ["druid"],