Skip to content

Commit

Permalink
xfail parquet lz4_raw tests for Spark 3.5.0 or later (#9157)
Browse files Browse the repository at this point in the history
Signed-off-by: Andy Grove <andygrove@nvidia.com>
  • Loading branch information
andygrove authored Sep 1, 2023
1 parent df016cb commit 9512bb2
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions integration_tests/src/main/python/parquet_testing_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from data_gen import copy_and_update
from pathlib import Path
import pytest
from spark_session import is_before_spark_330
from spark_session import is_before_spark_330, is_spark_350_or_later
import warnings

_rebase_confs = {
Expand All @@ -41,8 +41,6 @@
_error_files = {
"fixed_length_byte_array.parquet": "Exception",
"large_string_map.brotli.parquet": "Exception",
"lz4_raw_compressed.parquet": "Exception",
"lz4_raw_compressed_larger.parquet": "Exception",
"nation.dict-malformed.parquet": ("Exception", "https://github.com/NVIDIA/spark-rapids/issues/8644"),
"non_hadoop_lz4_compressed.parquet": "Exception",
"PARQUET-1481.parquet": "Exception",
Expand All @@ -66,6 +64,14 @@
if is_before_spark_330():
_xfail_files["rle_boolean_encoding.parquet"] = "Spark CPU cannot decode V2 style RLE before 3.3.x"

# Spark 3.5.0 adds support for lz4_raw compression codec, but we do not support that on GPU yet
if is_spark_350_or_later():
_xfail_files["lz4_raw_compressed.parquet"] = "https://github.com/NVIDIA/spark-rapids/issues/9156"
_xfail_files["lz4_raw_compressed_larger.parquet"] = "https://github.com/NVIDIA/spark-rapids/issues/9156"
else:
_error_files["lz4_raw_compressed.parquet"] = "Exception"
_error_files["lz4_raw_compressed_larger.parquet"] = "Exception"

def locate_parquet_testing_files():
"""
Finds the input files by first checking the standard input path,
Expand Down

0 comments on commit 9512bb2

Please sign in to comment.