[DO-NOT-MERGE][TESTS] Enable pandas API on Spark tests related to SPA…

…RK-43611
itholic · Jul 17, 2023 · d906e39 · d906e39
1 parent 12b4e73
commit d906e39
Show file tree

Hide file tree

Showing 29 changed files with 15 additions and 717 deletions.
diff --git a/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py b/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py
@@ -27,12 +27,6 @@ class FrameParityCombineTests(FrameCombineMixin, PandasOnSparkTestUtils, ReusedC
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_append(self):
-        super().test_append()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.computation.test_parity_combine import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py b/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py
@@ -27,22 +27,10 @@ class FrameParityComputeTests(FrameComputeMixin, PandasOnSparkTestUtils, ReusedC
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_diff(self):
-        super().test_diff()
-
     @unittest.skip("Spark Connect does not support RDD but the tests depend on them.")
     def test_mode(self):
         super().test_mode()
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_pct_change(self):
-        super().test_pct_change()
-
     @unittest.skip("TODO(SPARK-43618): Fix pyspark.sq.column._unary_op to work with Spark Connect.")
     def test_rank(self):
         super().test_rank()

diff --git a/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py b/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py
@@ -29,54 +29,6 @@ class FrameParityCumulativeTests(
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummax(self):
-        super().test_cummax()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummax_multiindex_columns(self):
-        super().test_cummax_multiindex_columns()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummin(self):
-        super().test_cummin()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummin_multiindex_columns(self):
-        super().test_cummin_multiindex_columns()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumprod(self):
-        super().test_cumprod()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumprod_multiindex_columns(self):
-        super().test_cumprod_multiindex_columns()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumsum(self):
-        super().test_cumsum()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumsum_multiindex_columns(self):
-        super().test_cumsum_multiindex_columns()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.computation.test_parity_cumulative import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py b/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py
@@ -29,36 +29,6 @@ class FrameParityMissingDataTests(
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_backfill(self):
-        super().test_backfill()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_bfill(self):
-        super().test_bfill()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_ffill(self):
-        super().test_ffill()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_fillna(self):
-        return super().test_fillna()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_pad(self):
-        super().test_pad()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.computation.test_parity_missing_data import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py b/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py
@@ -27,18 +27,6 @@ class FrameParityPivotTests(FramePivotMixin, PandasOnSparkTestUtils, ReusedConne
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_pivot_table(self):
-        super().test_pivot_table()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_pivot_table_dtypes(self):
-        super().test_pivot_table_dtypes()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.computation.test_parity_pivot import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py b/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py
@@ -24,23 +24,7 @@
 class DiffFramesParityBasicSlowTests(
     DiffFramesBasicSlowMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
 ):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_diff(self):
-        super().test_diff()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_rank(self):
-        super().test_rank()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_shift(self):
-        super().test_shift()
+    pass
 
 
 if __name__ == "__main__":

diff --git a/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py b/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py
@@ -27,12 +27,6 @@ class FrameParityReshapingTests(FrameReshapingMixin, PandasOnSparkTestUtils, Reu
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_transpose(self):
-        super().test_transpose()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.frame.test_parity_reshaping import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py b/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py
@@ -29,12 +29,6 @@ class FrameParityTimeSeriesTests(
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_shift(self):
-        super().test_shift()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.frame.test_parity_time_series import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py b/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py
@@ -24,35 +24,7 @@
 class GroupbyParityCumulativeTests(
     GroupbyCumulativeMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
 ):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumcount(self):
-        super().test_cumcount()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummax(self):
-        super().test_cummax()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummin(self):
-        super().test_cummin()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumprod(self):
-        super().test_cumprod()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumsum(self):
-        super().test_cumsum()
+    pass
 
 
 if __name__ == "__main__":

diff --git a/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py b/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py
@@ -24,23 +24,7 @@
 class GroupByParityTests(
     GroupByTestsMixin, PandasOnSparkTestUtils, TestUtils, ReusedConnectTestCase
 ):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_shift(self):
-        super().test_shift()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_diff(self):
-        super().test_diff()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_rank(self):
-        super().test_rank()
+    pass
 
 
 if __name__ == "__main__":

diff --git a/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py b/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py
@@ -24,23 +24,7 @@
 class GroupbyParityMissingDataTests(
     GroupbyMissingDataMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
 ):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_bfill(self):
-        super().test_bfill()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_ffill(self):
-        super().test_ffill()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_fillna(self):
-        super().test_fillna()
+    pass
 
 
 if __name__ == "__main__":

diff --git a/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py b/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py
@@ -29,18 +29,6 @@ class IndexesParityTests(
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_append(self):
-        super().test_append()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_monotonic(self):
-        super().test_monotonic()
-
     @unittest.skip("TODO(SPARK-43620): Support `Column` for SparkConnectColumn.__getitem__.")
     def test_factorize(self):
         super().test_factorize()

diff --git a/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py b/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py
@@ -29,12 +29,6 @@ class FrameParityResetIndexTests(
     def psdf(self):
         return ps.from_pandas(self.pdf)
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_reset_index_with_default_index_types(self):
-        super().test_reset_index_with_default_index_types()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.indexes.test_parity_reset_index import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py b/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py
@@ -22,11 +22,7 @@
 
 
 class SeriesParityArgOpsTests(SeriesArgOpsMixin, PandasOnSparkTestUtils, ReusedConnectTestCase):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_argsort(self):
-        super().test_argsort()
+    pass
 
 
 if __name__ == "__main__":

diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_compute.py b/python/pyspark/pandas/tests/connect/series/test_parity_compute.py
@@ -22,28 +22,10 @@
 
 
 class SeriesParityComputeTests(SeriesComputeMixin, PandasOnSparkTestUtils, ReusedConnectTestCase):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_diff(self):
-        super().test_diff()
-
     @unittest.skip("TODO(SPARK-43620): Support `Column` for SparkConnectColumn.__getitem__.")
     def test_factorize(self):
         super().test_factorize()
 
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_shift(self):
-        super().test_shift()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_unstack(self):
-        super().test_unstack()
-
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.connect.series.test_parity_compute import *  # noqa: F401

diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py b/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py
@@ -24,29 +24,7 @@
 class SeriesParityCumulativeTests(
     SeriesCumulativeMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
 ):
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummax(self):
-        super().test_cummax()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cummin(self):
-        super().test_cummin()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumprod(self):
-        super().test_cumprod()
-
-    @unittest.skip(
-        "TODO(SPARK-43611): Fix unexpected `AnalysisException` from Spark Connect client."
-    )
-    def test_cumsum(self):
-        super().test_cumsum()
+    pass
 
 
 if __name__ == "__main__":