Integration tests: Install specific fastparquet version. [databricks] (…

…#9607)

* Integration tests: Install specific fastparquet version.

Fixes #9603.

This commit changes the integration test setup to specifically install
fastparquet-0.8.3.

Prior to this change, when the fastparquet version is not specified, the pip
install caused 0.5.0 to be installed on some nodes, e.g. on Dataproc 2.0
(with Spark 3.1.1).
The older fastparquet versions do not support reading the contents of input
directories recursively, causing the tests to fail.

Note that this change doesn't bump the version all the way to 2023.8.0, so as
to preserve compatibility with Dataproc 2.0.  v0.8.3 seems to have the broadest
support.

Signed-off-by: MithunR <mythrocks@gmail.com>

* Switching to one specific fastparquet version.

* Added requirements.txt to the IT tar.gz package.

This should allow the installation of the right `fastparquet` version.

---------

Signed-off-by: MithunR <mythrocks@gmail.com>

integration_tests/requirements.txt

@@ -17,4 +17,4 @@ pandas
 pyarrow
 pytest-xdist >= 2.0.0
 findspark
-fastparquet >= 2023.8.0
+fastparquet == 0.8.3

integration_tests/src/assembly/bin.xml

@@ -42,6 +42,10 @@
             <source>${spark.rapids.source.basedir}/integration_tests/conftest.py</source>
             <outputDirectory>integration_tests</outputDirectory>
         </file>
+        <file>
+            <source>${spark.rapids.source.basedir}/integration_tests/requirements.txt</source>
+            <outputDirectory>integration_tests</outputDirectory>
+        </file>
         <file>
             <source>${project.build.directory}/extra-resources/rapids4spark-version-info.properties</source>
             <outputDirectory>integration_tests</outputDirectory>

jenkins/Dockerfile-blossom.integration.rocky

@@ -57,7 +57,7 @@ RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \
     conda install -y -c conda-forge sre_yield && \
     conda clean -ay
 # install pytest plugins for xdist parallel run
-RUN python -m pip install findspark pytest-xdist pytest-order fastparquet
+RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==0.8.3
 
 # Set default java as 1.8.0
 ENV JAVA_HOME "/usr/lib/jvm/java-1.8.0-openjdk"

jenkins/Dockerfile-blossom.integration.ubuntu

@@ -69,7 +69,7 @@ RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \
     conda install -y -c conda-forge sre_yield && \
     conda clean -ay
 # install pytest plugins for xdist parallel run
-RUN python -m pip install findspark pytest-xdist pytest-order fastparquet
+RUN python -m pip install findspark pytest-xdist pytest-order fastparquet==0.8.3
 
 RUN apt install -y inetutils-ping expect
 

jenkins/Dockerfile-blossom.ubuntu

@@ -58,7 +58,7 @@ RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64
 
 RUN ln -sfn /usr/bin/python3.8 /usr/bin/python
 RUN ln -sfn /usr/bin/python3.8 /usr/bin/python3
-RUN python -m pip install pytest sre_yield requests pandas pyarrow findspark pytest-xdist pre-commit pytest-order fastparquet
+RUN python -m pip install pytest sre_yield requests pandas pyarrow findspark pytest-xdist pre-commit pytest-order fastparquet==0.8.3
 
 # libnuma1 and libgomp1 are required by ucx packaging
 RUN apt install -y inetutils-ping expect wget libnuma1 libgomp1

jenkins/databricks/setup.sh

@@ -53,4 +53,4 @@ $PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES pytest sre_yield r
 
 # Install fastparquet (and numpy as its dependency).
 $PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES numpy
-$PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES fastparquet
+$PYSPARK_PYTHON -m pip install --target $PYTHON_SITE_PACKAGES fastparquet==0.8.3