add the new config file in the assembly

NVIDIA · Aug 1, 2024 · 8dbd354 · 8dbd354
1 parent 6ba7300
commit 8dbd354
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 32 deletions.
diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
@@ -78,6 +78,42 @@
     </dependencies>
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>clean-previously-populated-config-file</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>rm</executable>
+                            <arguments>
+                                <argument>-f</argument>
+                                <argument>${spark.rapids.source.basedir}/${rapids.module}/target/spark-rapids-default-configs.json</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>populate-default-configs-for-testing</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                        <configuration>
+                            <includeProjectDependencies>true</includeProjectDependencies>
+                            <mainClass>com.nvidia.spark.rapids.tests.DumpDefaultConfigs</mainClass>
+                            <classpathScope>provided</classpathScope>
+                            <arguments>
+                                <argument>json</argument>
+                                <argument>${spark.rapids.source.basedir}/${rapids.module}/target/spark-rapids-default-configs.json</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
                 <version>3.6.0</version>
@@ -180,36 +216,6 @@
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>exec-maven-plugin</artifactId>
                 <executions>
-                    <execution>
-                        <id>clean-previously-populated-config-file</id>
-                        <phase>verify</phase>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                        <configuration>
-                            <executable>rm</executable>
-                            <arguments>
-                                <argument>-f</argument>
-                                <argument>${spark.rapids.source.basedir}/${rapids.module}/target/spark-rapids-default-configs.json</argument>
-                            </arguments>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>populate-default-configs-for-testing</id>
-                        <phase>verify</phase>
-                        <goals>
-                            <goal>java</goal>
-                        </goals>
-                        <configuration>
-                            <includeProjectDependencies>true</includeProjectDependencies>
-                            <mainClass>com.nvidia.spark.rapids.tests.DumpDefaultConfigs</mainClass>
-                            <classpathScope>provided</classpathScope>
-                            <arguments>
-                                <argument>json</argument>
-                                <argument>${spark.rapids.source.basedir}/${rapids.module}/target/spark-rapids-default-configs.json</argument>
-                            </arguments>
-                        </configuration>
-                    </execution>
                     <execution>
                         <id>run-pyspark-tests</id>
                         <phase>verify</phase><!--run after packageing and collecting dependencies-->

diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
@@ -76,7 +76,7 @@ else
         # Make sure we have Parquet version >= 1.12 in the dependency
         LOWEST_PARQUET_JAR=$(echo -e "$MIN_PARQUET_JAR\n$PARQUET_HADOOP_TESTS" | sort -V | head -1)
         export INCLUDE_PARQUET_HADOOP_TEST_JAR=$([[ "$LOWEST_PARQUET_JAR" == "$MIN_PARQUET_JAR" ]] && echo true || echo false)
-        PLUGIN_JARS=$(echo "$TARGET_DIR"/../../dist/target/rapids-4-spark_*.jar)
+        PLUGIN_JARS=$(echo "$TARGET_DIR"/../../dist/rapids-4-spark_*.jar)
         # the integration-test-spark3xx.jar, should not include the integration-test-spark3xxtest.jar
         TEST_JARS=$(echo "$TARGET_DIR"/rapids-4-spark-integration-tests*-$INTEGRATION_TEST_VERSION.jar)
     fi
@@ -232,7 +232,7 @@ else
           "$TEST_ARGS"
           $RUN_TEST_PARAMS
           --junitxml=TEST-pytest-`date +%s%N`.xml
-          --default_configs_path="${SCRIPTPATH}/target/spark-rapids-default-configs.json"
+          --default_configs_path="${TARGET_DIR}/spark-rapids-default-configs.json"
           "$@")
 
     NUM_LOCAL_EXECS=${NUM_LOCAL_EXECS:-0}

diff --git a/integration_tests/src/assembly/bin.xml b/integration_tests/src/assembly/bin.xml
@@ -50,6 +50,10 @@
             <source>${project.build.outputDirectory}/rapids4spark-version-info.properties</source>
             <outputDirectory>integration_tests</outputDirectory>
         </file>
+        <file>
+            <source>${spark.rapids.source.basedir}/integration_tests/target/spark-rapids-default-configs.json</source>
+            <outputDirectory>integration_tests</outputDirectory>
+        </file>
     </files>
     <fileSets>
         <fileSet>