Skip to content

Commit

Permalink
[MINOR][PYTHON][TESTS] Move a test out of parity tests
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?
Move a test out of parity tests

### Why are the changes needed?
it is not tested in Spark Classic, not a parity test

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
ci

### Was this patch authored or co-authored using generative AI tooling?
no

Closes #46914 from zhengruifeng/move_a_non_parity_test.

Authored-by: Ruifeng Zheng <ruifengz@apache.org>
Signed-off-by: Ruifeng Zheng <ruifengz@apache.org>
  • Loading branch information
zhengruifeng committed Jun 7, 2024
1 parent 8911d59 commit 201df0d
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 24 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,29 @@


class SparkConnectDataFramePropertyTests(SparkConnectSQLTestCase):
def test_cached_property_is_copied(self):
schema = StructType(
[
StructField("id", IntegerType(), True),
StructField("name", StringType(), True),
StructField("age", IntegerType(), True),
StructField("city", StringType(), True),
]
)
# Create some dummy data
data = [
(1, "Alice", 30, "New York"),
(2, "Bob", 25, "San Francisco"),
(3, "Cathy", 29, "Los Angeles"),
(4, "David", 35, "Chicago"),
]
df = self.spark.createDataFrame(data, schema)
df_columns = df.columns
assert len(df.columns) == 4
for col in ["id", "name"]:
df_columns.remove(col)
assert len(df.columns) == 4

def test_cached_schema_to(self):
cdf = self.connect.read.table(self.tbl_name)
sdf = self.spark.read.table(self.tbl_name)
Expand Down
24 changes: 0 additions & 24 deletions python/pyspark/sql/tests/connect/test_parity_dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,37 +19,13 @@

from pyspark.sql.tests.test_dataframe import DataFrameTestsMixin
from pyspark.testing.connectutils import ReusedConnectTestCase
from pyspark.sql.types import StructType, StructField, IntegerType, StringType


class DataFrameParityTests(DataFrameTestsMixin, ReusedConnectTestCase):
def test_help_command(self):
df = self.spark.createDataFrame(data=[{"foo": "bar"}, {"foo": "baz"}])
super().check_help_command(df)

def test_cached_property_is_copied(self):
schema = StructType(
[
StructField("id", IntegerType(), True),
StructField("name", StringType(), True),
StructField("age", IntegerType(), True),
StructField("city", StringType(), True),
]
)
# Create some dummy data
data = [
(1, "Alice", 30, "New York"),
(2, "Bob", 25, "San Francisco"),
(3, "Cathy", 29, "Los Angeles"),
(4, "David", 35, "Chicago"),
]
df = self.spark.createDataFrame(data, schema)
df_columns = df.columns
assert len(df.columns) == 4
for col in ["id", "name"]:
df_columns.remove(col)
assert len(df.columns) == 4

@unittest.skip("Spark Connect does not support RDD but the tests depend on them.")
def test_toDF_with_schema_string(self):
super().test_toDF_with_schema_string()
Expand Down

0 comments on commit 201df0d

Please sign in to comment.