feat(pyspark): implement Distinct for pyspark

ibis-project · Apr 9, 2022 · 4306ad9 · 4306ad9
1 parent 3f48cb8
commit 4306ad9
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py
@@ -1176,6 +1176,12 @@ def compile_join(t, expr, scope, timecontext, *, how):
     return left_df.join(right_df, pred_columns, how)
 
 
+@compiles(ops.Distinct)
+def compile_distinct(t, expr, scope, timecontext):
+    op = expr.op()
+    return t.translate(op.table, scope, timecontext).distinct()
+
+
 def _canonicalize_interval(t, interval, scope, timecontext, **kwargs):
     """Convert interval to integer timestamp of second
 

diff --git a/ibis/tests/sql/test_compiler.py b/ibis/tests/sql/test_compiler.py
@@ -71,7 +71,7 @@ def test_table_distinct(con):
 
 def test_column_distinct(con):
     t = con.table('functional_alltypes')
-    expr = t[["string_col"]].distinct()
+    expr = t[t.string_col].distinct()
 
     result = Compiler.to_sql(expr)
     expected = """SELECT DISTINCT `string_col`

diff --git a/ibis/tests/sql/test_sqlalchemy.py b/ibis/tests/sql/test_sqlalchemy.py
@@ -656,7 +656,7 @@ def test_not_exists(con, not_exists):
             ).distinct(),
         ),
         (
-            lambda t: t[["string_col"]].distinct(),
+            lambda t: t[t.string_col].distinct(),
             lambda sat: sa.select([sat.c.string_col.distinct()]),
         ),
         (