From 16729e441bc0a4ba05c471ec2271c4ead41b8094 Mon Sep 17 00:00:00 2001
From: Jihoon Son <ghoonson@gmail.com>
Date: Fri, 16 Aug 2024 15:42:50 -0700
Subject: [PATCH] Revert "Fix the mismatching default configs in integration
 tests  (#11283)"

This reverts commit bc8c577b7b21b115143814cbd3ca83ab948e55c7.

Signed-off-by: Jihoon Son <ghoonson@gmail.com>
---
 integration_tests/conftest.py                 |   5 +-
 integration_tests/pom.xml                     |  37 +-----
 integration_tests/run_pyspark_from_build.sh   |   2 -
 integration_tests/src/assembly/bin.xml        |   6 +-
 .../src/main/python/arithmetic_ops_test.py    | 113 ++++++------------
 integration_tests/src/main/python/ast_test.py |   8 +-
 .../src/main/python/cast_test.py              |  27 ++---
 integration_tests/src/main/python/conftest.py |   6 -
 .../src/main/python/date_time_test.py         |   3 +-
 .../src/main/python/hash_aggregate_test.py    |  13 +-
 .../src/main/python/spark_session.py          |  27 +++--
 .../src/main/python/string_test.py            |   3 +-
 .../rapids/tests/DumpDefaultConfigs.scala     |  35 ------
 pom.xml                                       |  16 +--
 scala2.13/integration_tests/pom.xml           |  37 +-----
 scala2.13/pom.xml                             |  16 +--
 .../com/nvidia/spark/rapids/RapidsConf.scala  |  74 +-----------
 17 files changed, 85 insertions(+), 343 deletions(-)
 delete mode 100644 integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/DumpDefaultConfigs.scala

diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
index 1587b6591bd..f2e3435d0c5 100644
--- a/integration_tests/conftest.py
+++ b/integration_tests/conftest.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2024, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -62,6 +62,3 @@ def pytest_addoption(parser):
     parser.addoption(
         "--pyarrow_test", action='store_true', default=False, help="if enable pyarrow tests"
     )
-    parser.addoption(
-        "--default_configs_path", action="store", default=None, help="path to a JSON file that stores default configs for integration test"
-    )
diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
index 8829cd2d6ec..3ea20b75610 100644
--- a/integration_tests/pom.xml
+++ b/integration_tests/pom.xml
@@ -68,16 +68,6 @@
             <!-- for hive udf test cases -->
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-databind</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.github.scopt/scopt -->
         <dependency>
@@ -88,31 +78,6 @@
     </dependencies>
     <build>
         <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>populate-default-configs-for-testing</id>
-                        <phase>generate-test-resources</phase>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <configuration>
-                            <target>
-                                <taskdef resource="net/sf/antcontrib/antcontrib.properties"/>
-                                <java classname="com.nvidia.spark.rapids.tests.DumpDefaultConfigs"
-                                    failonerror="true"
-                                    classpathref="maven.compile.classpath"
-                                    fork="true">
-                                    <arg value="json"/>
-                                    <arg value="${project.build.directory}/spark-rapids-default-configs.json"/>
-                                </java>
-                            </target>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
                 <version>3.6.0</version>
@@ -216,7 +181,7 @@
                 <artifactId>exec-maven-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>run-pyspark-tests</id>
+                        <id>run pyspark tests</id>
                         <phase>verify</phase><!--run after packageing and collecting dependencies-->
                         <goals>
                             <goal>exec</goal>
diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
index cd6cc5cef00..22a23349791 100755
--- a/integration_tests/run_pyspark_from_build.sh
+++ b/integration_tests/run_pyspark_from_build.sh
@@ -223,7 +223,6 @@ else
 
     REPORT_CHARS=${REPORT_CHARS:="fE"} # default as (f)ailed, (E)rror
     STD_INPUT_PATH="$INPUT_PATH"/src/test/resources
-    DEFAULT_CONFIGS_PATH=${DEFAULT_CONFIGS_PATH:-${TARGET_DIR}/spark-rapids-default-configs.json}
     TEST_COMMON_OPTS=(-v
           -r"$REPORT_CHARS"
           "$TEST_TAGS"
@@ -233,7 +232,6 @@ else
           "$TEST_ARGS"
           $RUN_TEST_PARAMS
           --junitxml=TEST-pytest-`date +%s%N`.xml
-          --default_configs_path="${DEFAULT_CONFIGS_PATH}"
           "$@")
 
     NUM_LOCAL_EXECS=${NUM_LOCAL_EXECS:-0}
diff --git a/integration_tests/src/assembly/bin.xml b/integration_tests/src/assembly/bin.xml
index 29f9c15fb88..6209d0b152a 100644
--- a/integration_tests/src/assembly/bin.xml
+++ b/integration_tests/src/assembly/bin.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2024, NVIDIA CORPORATION.
+  Copyright (c) 2020-2023, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -50,10 +50,6 @@
             <source>${project.build.outputDirectory}/rapids4spark-version-info.properties</source>
             <outputDirectory>integration_tests</outputDirectory>
         </file>
-        <file>
-            <source>${project.build.directory}/spark-rapids-default-configs.json</source>
-            <outputDirectory>integration_tests</outputDirectory>
-        </file>
     </files>
     <fileSets>
         <fileSet>
diff --git a/integration_tests/src/main/python/arithmetic_ops_test.py b/integration_tests/src/main/python/arithmetic_ops_test.py
index 9943d7deb7e..d7fd941b97b 100644
--- a/integration_tests/src/main/python/arithmetic_ops_test.py
+++ b/integration_tests/src/main/python/arithmetic_ops_test.py
@@ -92,8 +92,6 @@
         pytest.param(_decimal_gen_38_0, marks=pytest.mark.skipif(
             is_spark_330_or_later(), reason='This case overflows in Spark 3.3.0+'))]
 
-allow_neg_scale_conf = {'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'}
-
 def _get_overflow_df(spark, data, data_type, expr):
     return spark.createDataFrame(
         SparkContext.getOrCreate().parallelize([data]),
@@ -110,8 +108,7 @@ def test_addition(data_gen):
                 f.lit(-12).cast(data_type) + f.col('b'),
                 f.lit(None).cast(data_type) + f.col('a'),
                 f.col('b') + f.lit(None).cast(data_type),
-                f.col('a') + f.col('b')),
-    conf=allow_neg_scale_conf)
+                f.col('a') + f.col('b')))
 
 # If it will not overflow for multiply it is good for add too
 @pytest.mark.parametrize('data_gen', _no_overflow_multiply_gens, ids=idfn)
@@ -136,8 +133,7 @@ def test_subtraction(data_gen):
                 f.lit(-12).cast(data_type) - f.col('b'),
                 f.lit(None).cast(data_type) - f.col('a'),
                 f.col('b') - f.lit(None).cast(data_type),
-                f.col('a') - f.col('b')),
-    conf=allow_neg_scale_conf)
+                f.col('a') - f.col('b')))
 
 @pytest.mark.parametrize('lhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(6, 5),
     DecimalGen(6, 4), DecimalGen(5, 4), DecimalGen(5, 3), DecimalGen(4, 2), DecimalGen(3, -2),
@@ -149,8 +145,7 @@ def test_subtraction(data_gen):
 @disable_ansi_mode
 def test_addition_subtraction_mixed(lhs, rhs, addOrSub):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"a {addOrSub} b"),
-        conf=allow_neg_scale_conf
+        lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"a {addOrSub} b")
     )
 
 # If it will not overflow for multiply it is good for subtract too
@@ -183,8 +178,7 @@ def test_multiplication(data_gen):
                 f.lit(None).cast(data_type) * f.col('a'),
                 f.col('b') * f.lit(None).cast(data_type),
                 f.col('a') * f.col('b')
-                ),
-    conf=allow_neg_scale_conf)
+                ))
 
 @allow_non_gpu('ProjectExec', 'Alias', 'Multiply', 'Cast')
 @pytest.mark.parametrize('data_gen', _no_overflow_multiply_gens_for_fallback, ids=idfn)
@@ -221,18 +215,16 @@ def test_multiplication_ansi_overflow():
 def test_multiplication_mixed(lhs, rhs):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : two_col_df(spark, lhs, rhs).select(
-                f.col('a') * f.col('b')),
-    conf=allow_neg_scale_conf)
+                f.col('a') * f.col('b')))
 
 @approximate_float # we should get the perfectly correct answer for floats except when casting a decimal to a float in some corner cases.
 @pytest.mark.parametrize('lhs', [float_gen, double_gen], ids=idfn)
 @pytest.mark.parametrize('rhs', [DecimalGen(6, 3), DecimalGen(10, -2), DecimalGen(15, 3)], ids=idfn)
 def test_float_multiplication_mixed(lhs, rhs):
-    conf = copy_and_update(allow_neg_scale_conf, {'spark.rapids.sql.castDecimalToFloat.enabled': 'true'})
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : two_col_df(spark, lhs, rhs).select(
                 f.col('a') * f.col('b')),
-            conf=conf)
+            conf={'spark.rapids.sql.castDecimalToFloat.enabled': 'true'})
 
 @pytest.mark.parametrize('data_gen', [double_gen, decimal_gen_32bit_neg_scale, DecimalGen(6, 3),
  DecimalGen(5, 5), DecimalGen(6, 0), DecimalGen(7, 4), DecimalGen(15, 0), DecimalGen(18, 0),
@@ -246,8 +238,7 @@ def test_division(data_gen):
                 f.lit(-12).cast(data_type) / f.col('b'),
                 f.lit(None).cast(data_type) / f.col('a'),
                 f.col('b') / f.lit(None).cast(data_type),
-                f.col('a') / f.col('b')),
-        conf=allow_neg_scale_conf)
+                f.col('a') / f.col('b')))
 
 @pytest.mark.parametrize('rhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(4, 1), DecimalGen(5, 0), DecimalGen(5, 1), DecimalGen(10, 5)], ids=idfn)
 @pytest.mark.parametrize('lhs', [byte_gen, short_gen, int_gen, long_gen, DecimalGen(5, 3), DecimalGen(4, 2), DecimalGen(1, -2), DecimalGen(16, 1)], ids=idfn)
@@ -256,8 +247,7 @@ def test_division_mixed(lhs, rhs):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : two_col_df(spark, lhs, rhs).select(
                 f.col('a'), f.col('b'),
-                f.col('a') / f.col('b')),
-        conf=allow_neg_scale_conf)
+                f.col('a') / f.col('b')))
 
 # Spark has some problems with some decimal operations where it can try to generate a type that is invalid (scale > precision) which results in an error
 # instead of increasing the precision. So we have a second test that deals with a few of these use cases
@@ -268,8 +258,7 @@ def test_division_mixed_larger_dec(lhs, rhs):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : two_col_df(spark, lhs, rhs).select(
                 f.col('a'), f.col('b'),
-                f.col('a') / f.col('b')),
-    conf=allow_neg_scale_conf)
+                f.col('a') / f.col('b')))
 
 @disable_ansi_mode
 def test_special_decimal_division():
@@ -279,19 +268,17 @@ def test_special_decimal_division():
             data_gen = DecimalGen(precision, scale)
             assert_gpu_and_cpu_are_equal_collect(
                     lambda spark : two_col_df(spark, data_gen, data_gen).select(
-                        f.col('a') / f.col('b')),
-                conf=allow_neg_scale_conf)
+                        f.col('a') / f.col('b')))
 
 @approximate_float # we should get the perfectly correct answer for floats except when casting a decimal to a float in some corner cases.
 @pytest.mark.parametrize('rhs', [float_gen, double_gen], ids=idfn)
 @pytest.mark.parametrize('lhs', [DecimalGen(5, 3), DecimalGen(4, 2), DecimalGen(1, -2), DecimalGen(16, 1)], ids=idfn)
 @disable_ansi_mode
 def test_float_division_mixed(lhs, rhs):
-    conf = copy_and_update(allow_neg_scale_conf, {'spark.rapids.sql.castDecimalToFloat.enabled': 'true'})
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : two_col_df(spark, lhs, rhs).select(
                 f.col('a') / f.col('b')),
-        conf=conf)
+            conf={'spark.rapids.sql.castDecimalToFloat.enabled': 'true'})
 
 @pytest.mark.parametrize('data_gen', integral_gens + [
     decimal_gen_32bit, decimal_gen_64bit, _decimal_gen_7_7, _decimal_gen_18_3, _decimal_gen_30_2,
@@ -314,8 +301,7 @@ def test_int_division(data_gen):
 def test_int_division_mixed(lhs, rhs):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : two_col_df(spark, lhs, rhs).selectExpr(
-                'a DIV b'),
-        conf=allow_neg_scale_conf)
+                'a DIV b'))
 
 @pytest.mark.parametrize('data_gen', _arith_data_gens, ids=idfn)
 @disable_ansi_mode
@@ -327,8 +313,7 @@ def test_mod(data_gen):
                 f.lit(-12).cast(data_type) % f.col('b'),
                 f.lit(None).cast(data_type) % f.col('a'),
                 f.col('b') % f.lit(None).cast(data_type),
-                f.col('a') % f.col('b')),
-        conf=allow_neg_scale_conf)
+                f.col('a') % f.col('b')))
 
 # pmod currently falls back for Decimal(precision=38)
 # https://github.com/NVIDIA/spark-rapids/issues/6336
@@ -449,8 +434,7 @@ def test_mod_pmod_by_zero_not_ansi(data_gen):
 @disable_ansi_mode
 def test_mod_mixed(lhs, rhs):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"a % b"),
-        conf=allow_neg_scale_conf)
+        lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"a % b"))
 
 # @pytest.mark.skipif(not is_databricks113_or_later() and not is_spark_340_or_later(), reason="https://github.com/NVIDIA/spark-rapids/issues/8330")
 @pytest.mark.parametrize('lhs', [DecimalGen(38,0), DecimalGen(37,2), DecimalGen(38,5), DecimalGen(38,-10), DecimalGen(38,7)], ids=idfn)
@@ -458,8 +442,7 @@ def test_mod_mixed(lhs, rhs):
 @disable_ansi_mode
 def test_mod_mixed_decimal128(lhs, rhs):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : two_col_df(spark, lhs, rhs).selectExpr("a", "b", f"a % b"),
-        conf=allow_neg_scale_conf)
+        lambda spark : two_col_df(spark, lhs, rhs).selectExpr("a", "b", f"a % b"))
 
 # Split into 4 tests to permute https://github.com/NVIDIA/spark-rapids/issues/7553 failures
 @pytest.mark.parametrize('lhs', [byte_gen, short_gen, int_gen, long_gen], ids=idfn)
@@ -478,8 +461,7 @@ def test_pmod_mixed_numeric(lhs, rhs):
 def test_pmod_mixed_decimal_lhs(lhs, rhs):
     assert_gpu_fallback_collect(
         lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)"),
-        "Pmod",
-        conf=allow_neg_scale_conf)
+        "Pmod")
 
 @allow_non_gpu("ProjectExec", "Pmod")
 @pytest.mark.parametrize('lhs', [byte_gen, short_gen, int_gen, long_gen], ids=idfn)
@@ -490,8 +472,7 @@ def test_pmod_mixed_decimal_lhs(lhs, rhs):
 def test_pmod_mixed_decimal_rhs(lhs, rhs):
     assert_gpu_fallback_collect(
         lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)"),
-        "Pmod",
-        conf=allow_neg_scale_conf)
+        "Pmod")
 
 @allow_non_gpu("ProjectExec", "Pmod")
 @pytest.mark.parametrize('lhs', [DecimalGen(6, 5), DecimalGen(6, 4), DecimalGen(5, 4), DecimalGen(5, 3),
@@ -504,8 +485,7 @@ def test_pmod_mixed_decimal_rhs(lhs, rhs):
 def test_pmod_mixed_decimal(lhs, rhs):
     assert_gpu_fallback_collect(
         lambda spark : two_col_df(spark, lhs, rhs).selectExpr(f"pmod(a, b)"),
-        "Pmod",
-    conf=allow_neg_scale_conf)
+        "Pmod")
 
 @pytest.mark.parametrize('data_gen', double_gens, ids=idfn)
 def test_signum(data_gen):
@@ -516,31 +496,27 @@ def test_signum(data_gen):
 @disable_ansi_mode
 def test_unary_minus(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).selectExpr('-a'),
-        conf=allow_neg_scale_conf)
+            lambda spark : unary_op_df(spark, data_gen).selectExpr('-a'))
 
 @pytest.mark.parametrize('data_gen', _arith_decimal_gens_high_precision, ids=idfn)
 @pytest.mark.skipif(is_scala213(), reason="Apache Spark built with Scala 2.13 produces inconsistent results at high precision (SPARK-45438)")
 def test_unary_minus_decimal128(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).selectExpr('-a'),
-        conf=allow_neg_scale_conf)
+            lambda spark : unary_op_df(spark, data_gen).selectExpr('-a'))
 
 @pytest.mark.parametrize('data_gen', _no_overflow_multiply_gens + [float_gen, double_gen] +
     _arith_decimal_gens_low_precision, ids=idfn)
 def test_unary_minus_ansi_no_overflow(data_gen):
-    conf = copy_and_update(ansi_enabled_conf, allow_neg_scale_conf)
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).selectExpr('-a'),
-            conf=conf)
+            conf=ansi_enabled_conf)
 
 @pytest.mark.parametrize('data_gen', _arith_decimal_gens_high_precision, ids=idfn)
 @pytest.mark.skipif(is_scala213(), reason="Apache Spark built with Scala 2.13 produces inconsistent results at high precision (SPARK-45438)")
 def test_unary_minus_ansi_no_overflow_decimal128(data_gen):
-    conf = copy_and_update(ansi_enabled_conf, allow_neg_scale_conf)
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).selectExpr('-a'),
-            conf=conf)
+            conf=ansi_enabled_conf)
 
 @pytest.mark.parametrize('data_type,value', [
     (LongType(), LONG_MIN),
@@ -564,39 +540,34 @@ def test_unary_minus_ansi_overflow(data_type, value):
 @pytest.mark.parametrize('data_gen', _arith_data_gens, ids=idfn)
 def test_unary_positive(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : unary_op_df(spark, data_gen).selectExpr('+a'),
-    conf=allow_neg_scale_conf)
+        lambda spark : unary_op_df(spark, data_gen).selectExpr('+a'))
 
 @pytest.mark.parametrize('data_gen', numeric_gens + _arith_decimal_gens_low_precision, ids=idfn)
 @disable_ansi_mode
 def test_abs(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).selectExpr('abs(a)'),
-        conf=allow_neg_scale_conf)
+            lambda spark : unary_op_df(spark, data_gen).selectExpr('abs(a)'))
 
 @pytest.mark.parametrize('data_gen', _arith_decimal_gens_high_precision, ids=idfn)
 @pytest.mark.skipif(is_scala213(), reason="Apache Spark built with Scala 2.13 produces inconsistent results at high precision (SPARK-45438)")
 def test_abs_decimal128(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).selectExpr('abs(a)'),
-        conf=allow_neg_scale_conf)
+            lambda spark : unary_op_df(spark, data_gen).selectExpr('abs(a)'))
 
 # ANSI is ignored for abs prior to 3.2.0, but still okay to test it a little more.
 @pytest.mark.parametrize('data_gen', _no_overflow_multiply_gens + [float_gen, double_gen] +
     _arith_decimal_gens_low_precision, ids=idfn)
 def test_abs_ansi_no_overflow(data_gen):
-    conf = copy_and_update(ansi_enabled_conf, allow_neg_scale_conf)
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).selectExpr('abs(a)'),
-            conf=conf)
+            conf=ansi_enabled_conf)
 
 @pytest.mark.parametrize('data_gen', _arith_decimal_gens_high_precision, ids=idfn)
 @pytest.mark.skipif(is_scala213(), reason="Apache Spark built with Scala 2.13 produces inconsistent results at high precision")
 def test_abs_ansi_no_overflow_decimal128(data_gen):
-    conf = copy_and_update(ansi_enabled_conf, allow_neg_scale_conf)
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).selectExpr('a','abs(a)'),
-            conf=conf)
+            conf=ansi_enabled_conf)
 
 # Only run this test for Spark v3.2.0 and later to verify abs will
 # throw exceptions for overflow when ANSI mode is enabled.
@@ -673,11 +644,11 @@ def test_floor_ceil_overflow(data_gen):
         "pyspark.errors.exceptions.captured.ArithmeticException: [NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION]"
     assert_gpu_and_cpu_error(
         lambda spark: unary_op_df(spark, data_gen).selectExpr('floor(a)').collect(),
-        conf=allow_neg_scale_conf,
+        conf={},
         error_message=exception_type)
     assert_gpu_and_cpu_error(
         lambda spark: unary_op_df(spark, data_gen).selectExpr('ceil(a)').collect(),
-        conf=allow_neg_scale_conf,
+        conf={},
         error_message=exception_type)
 
 @pytest.mark.parametrize('data_gen', double_gens, ids=idfn)
@@ -744,8 +715,7 @@ def test_decimal_bround(data_gen):
                 'bround(a, -1)',
                 'bround(a, 1)',
                 'bround(a, 2)',
-                'bround(a, 10)'),
-    conf=allow_neg_scale_conf)
+                'bround(a, 10)'))
 
 @incompat
 @approximate_float
@@ -760,8 +730,7 @@ def test_decimal_round(data_gen):
                 'round(a, -1)',
                 'round(a, 1)',
                 'round(a, 2)',
-                'round(a, 10)'),
-    conf=allow_neg_scale_conf)
+                'round(a, 10)'))
 
 
 @incompat
@@ -897,8 +866,7 @@ def test_cosh(data_gen):
 @pytest.mark.parametrize('data_gen', double_gens, ids=idfn)
 def test_acosh(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).selectExpr('acosh(a)'),
-    conf={'spark.rapids.sql.improvedFloatOps.enabled': 'false'})
+            lambda spark : unary_op_df(spark, data_gen).selectExpr('acosh(a)'))
 
 # The default approximate is 1e-6 or 1 in a million
 # in some cases we need to adjust this because the algorithm is different
@@ -909,7 +877,7 @@ def test_acosh(data_gen):
 def test_columnar_acosh_improved(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).selectExpr('acosh(a)'),
-        {'spark.rapids.sql.improvedFloatOps.enabled': 'true'})
+            {'spark.rapids.sql.improvedFloatOps.enabled': 'true'})
 
 @approximate_float
 @pytest.mark.parametrize('data_gen', double_gens, ids=idfn)
@@ -933,8 +901,7 @@ def test_asin(data_gen):
 @pytest.mark.parametrize('data_gen', double_gens, ids=idfn)
 def test_asinh(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).selectExpr('asinh(a)'),
-        conf={'spark.rapids.sql.improvedFloatOps.enabled': 'false'})
+            lambda spark : unary_op_df(spark, data_gen).selectExpr('asinh(a)'))
 
 # The default approximate is 1e-6 or 1 in a million
 # in some cases we need to adjust this because the algorithm is different
@@ -945,7 +912,7 @@ def test_asinh(data_gen):
 def test_columnar_asinh_improved(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).selectExpr('asinh(a)'),
-        conf={'spark.rapids.sql.improvedFloatOps.enabled': 'true'})
+            {'spark.rapids.sql.improvedFloatOps.enabled': 'true'})
 
 @approximate_float
 @pytest.mark.parametrize('data_gen', double_gens, ids=idfn)
@@ -1050,7 +1017,7 @@ def test_columnar_pow(data_gen):
 def test_least(data_gen):
     num_cols = 20
     s1 = with_cpu_session(
-        lambda spark: gen_scalar(data_gen, force_no_nulls=not isinstance(data_gen, NullGen)), conf=allow_neg_scale_conf)
+        lambda spark: gen_scalar(data_gen, force_no_nulls=not isinstance(data_gen, NullGen)))
     # we want lots of nulls
     gen = StructGen([('_c' + str(x), data_gen.copy_special_case(None, weight=100.0))
         for x in range(0, num_cols)], nullable=False)
@@ -1060,14 +1027,13 @@ def test_least(data_gen):
     data_type = data_gen.data_type
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : gen_df(spark, gen).select(
-                f.least(*command_args)),
-    conf=allow_neg_scale_conf)
+                f.least(*command_args)))
 
 @pytest.mark.parametrize('data_gen', all_basic_gens + _arith_decimal_gens, ids=idfn)
 def test_greatest(data_gen):
     num_cols = 20
     s1 = with_cpu_session(
-        lambda spark: gen_scalar(data_gen, force_no_nulls=not isinstance(data_gen, NullGen)), conf=allow_neg_scale_conf)
+        lambda spark: gen_scalar(data_gen, force_no_nulls=not isinstance(data_gen, NullGen)))
     # we want lots of nulls
     gen = StructGen([('_c' + str(x), data_gen.copy_special_case(None, weight=100.0))
         for x in range(0, num_cols)], nullable=False)
@@ -1076,8 +1042,7 @@ def test_greatest(data_gen):
     data_type = data_gen.data_type
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : gen_df(spark, gen).select(
-                f.greatest(*command_args)),
-    conf=allow_neg_scale_conf)
+                f.greatest(*command_args)))
 
 
 def _test_div_by_zero(ansi_mode, expr, is_lit=False):
diff --git a/integration_tests/src/main/python/ast_test.py b/integration_tests/src/main/python/ast_test.py
index 7e9c91e4a47..ed6547a429b 100644
--- a/integration_tests/src/main/python/ast_test.py
+++ b/integration_tests/src/main/python/ast_test.py
@@ -191,14 +191,12 @@ def test_atan(data_descr):
 @approximate_float
 @pytest.mark.parametrize('data_descr', [(double_gen, False)], ids=idfn)
 def test_asinh(data_descr):
-    assert_unary_ast(data_descr, lambda df: df.selectExpr('asinh(a)'),
-                     conf={'spark.rapids.sql.improvedFloatOps.enabled': 'false'})
+    assert_unary_ast(data_descr, lambda df: df.selectExpr('asinh(a)'))
 
 @approximate_float
 @pytest.mark.parametrize('data_descr', ast_double_descr, ids=idfn)
 def test_acosh(data_descr):
-    assert_unary_ast(data_descr, lambda df: df.selectExpr('acosh(a)'),
-                     conf={'spark.rapids.sql.improvedFloatOps.enabled': 'false'})
+    assert_unary_ast(data_descr, lambda df: df.selectExpr('acosh(a)'))
 
 @approximate_float
 @pytest.mark.parametrize('data_descr', ast_double_descr, ids=idfn)
@@ -213,7 +211,7 @@ def test_atanh(data_descr):
 @pytest.mark.parametrize('data_descr', [(DoubleGen(min_exp=-20, max_exp=20), True)], ids=idfn)
 def test_asinh_improved(data_descr):
     assert_unary_ast(data_descr, lambda df: df.selectExpr('asinh(a)'),
-                     conf={'spark.rapids.sql.improvedFloatOps.enabled': 'true'})
+        conf={'spark.rapids.sql.improvedFloatOps.enabled': 'true'})
 
 # The default approximate is 1e-6 or 1 in a million
 # in some cases we need to adjust this because the algorithm is different
diff --git a/integration_tests/src/main/python/cast_test.py b/integration_tests/src/main/python/cast_test.py
index ec22253a9a3..f7784178182 100644
--- a/integration_tests/src/main/python/cast_test.py
+++ b/integration_tests/src/main/python/cast_test.py
@@ -194,8 +194,7 @@ def test_cast_string_timestamp_fallback():
 def test_cast_decimal_to(data_gen, to_type):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).select(f.col('a').cast(to_type), f.col('a')),
-            conf = {'spark.rapids.sql.castDecimalToFloat.enabled': 'true',
-                    'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            conf = {'spark.rapids.sql.castDecimalToFloat.enabled': 'true'})
 
 @approximate_float
 @pytest.mark.parametrize('data_gen', [
@@ -209,8 +208,7 @@ def test_ansi_cast_decimal_to(data_gen, to_type):
     assert_gpu_and_cpu_are_equal_collect(
             lambda spark : unary_op_df(spark, data_gen).select(f.col('a').cast(to_type), f.col('a')),
             conf = {'spark.rapids.sql.castDecimalToFloat.enabled': True,
-                    'spark.sql.ansi.enabled': True,
-                    'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+                'spark.sql.ansi.enabled': True})
 
 @datagen_overrides(seed=0, reason='https://github.com/NVIDIA/spark-rapids/issues/10050')
 @pytest.mark.parametrize('data_gen', [
@@ -230,8 +228,7 @@ def test_ansi_cast_decimal_to(data_gen, to_type):
     DecimalType(1, -1)], ids=meta_idfn('to:'))
 def test_cast_decimal_to_decimal(data_gen, to_type):
     assert_gpu_and_cpu_are_equal_collect(
-            lambda spark : unary_op_df(spark, data_gen).select(f.col('a').cast(to_type), f.col('a')),
-        conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            lambda spark : unary_op_df(spark, data_gen).select(f.col('a').cast(to_type), f.col('a')))
 
 @pytest.mark.parametrize('data_gen', [byte_gen, short_gen, int_gen, long_gen], ids=idfn)
 @pytest.mark.parametrize('to_type', [
@@ -251,26 +248,22 @@ def test_cast_integral_to_decimal(data_gen, to_type):
 def test_cast_byte_to_decimal_overflow():
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark : unary_op_df(spark, byte_gen).select(
-            f.col('a').cast(DecimalType(2, -1))),
-        conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            f.col('a').cast(DecimalType(2, -1))))
 
 def test_cast_short_to_decimal_overflow():
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark : unary_op_df(spark, short_gen).select(
-            f.col('a').cast(DecimalType(4, -1))),
-        conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            f.col('a').cast(DecimalType(4, -1))))
 
 def test_cast_int_to_decimal_overflow():
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark : unary_op_df(spark, int_gen).select(
-            f.col('a').cast(DecimalType(9, -1))),
-        conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            f.col('a').cast(DecimalType(9, -1))))
 
 def test_cast_long_to_decimal_overflow():
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark : unary_op_df(spark, long_gen).select(
-            f.col('a').cast(DecimalType(18, -1))),
-    conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            f.col('a').cast(DecimalType(18, -1))))
 
 
 _float_special_cases = [(float("inf"), 5.0), (float("-inf"), 5.0), (float("nan"), 5.0)]
@@ -289,8 +282,7 @@ def test_cast_floating_point_to_decimal(data_gen, to_type):
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark : unary_op_df(spark, data_gen).select(
             f.col('a'), f.col('a').cast(to_type)),
-        conf={'spark.rapids.sql.castFloatToDecimal.enabled': 'true',
-              'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+        conf={'spark.rapids.sql.castFloatToDecimal.enabled': 'true'})
 
 # casting these types to string should be passed
 basic_gens_for_cast_to_string = [ByteGen, ShortGen, IntegerGen, LongGen, StringGen, BooleanGen, DateGen, TimestampGen]
@@ -447,8 +439,7 @@ def is_neg_dec_scale_bug_version():
 def test_cast_string_to_negative_scale_decimal():
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark: unary_op_df(spark, StringGen("[0-9]{9}")).select(
-            f.col('a').cast(DecimalType(8, -3))),
-    conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+            f.col('a').cast(DecimalType(8, -3))))
 
 @pytest.mark.skipif(is_before_spark_330(), reason="ansi cast throws exception only in 3.3.0+")
 @pytest.mark.parametrize('type', [DoubleType(), FloatType()], ids=idfn)
diff --git a/integration_tests/src/main/python/conftest.py b/integration_tests/src/main/python/conftest.py
index 1503c024fe8..49f69f31837 100644
--- a/integration_tests/src/main/python/conftest.py
+++ b/integration_tests/src/main/python/conftest.py
@@ -156,10 +156,6 @@ def is_parquet_testing_tests_forced():
 def get_inject_oom_conf():
     return _inject_oom
 
-_default_configs_path = None
-
-def get_default_configs_path():
-    return _default_configs_path
 
 # For datagen: we expect a seed to be provided by the environment, or default to 0.
 # Note that tests can override their seed when calling into datagen by setting seed= in their tests.
@@ -308,8 +304,6 @@ def pytest_configure(config):
         raise Exception("not supported test type {}".format(test_type))
     global _is_parquet_testing_tests_forced
     _is_parquet_testing_tests_forced = config.getoption("force_parquet_testing_tests")
-    global _default_configs_path
-    _default_configs_path = config.getoption("default_configs_path")
 
 # For OOM injection: we expect a seed to be provided by the environment, or default to 1.
 # This is done such that any worker started by the xdist plugin for pytest will
diff --git a/integration_tests/src/main/python/date_time_test.py b/integration_tests/src/main/python/date_time_test.py
index 34678a2e1ad..0c877f00238 100644
--- a/integration_tests/src/main/python/date_time_test.py
+++ b/integration_tests/src/main/python/date_time_test.py
@@ -622,8 +622,7 @@ def test_unsupported_fallback_to_date():
 @allow_non_gpu(*non_utc_allow)
 def test_timestamp_seconds(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : unary_op_df(spark, data_gen).selectExpr("timestamp_seconds(a)"),
-    conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+        lambda spark : unary_op_df(spark, data_gen).selectExpr("timestamp_seconds(a)"))
 
 @allow_non_gpu(*non_utc_allow)
 def test_timestamp_seconds_long_overflow():
diff --git a/integration_tests/src/main/python/hash_aggregate_test.py b/integration_tests/src/main/python/hash_aggregate_test.py
index 5f1470d4d9c..ea429d4533c 100644
--- a/integration_tests/src/main/python/hash_aggregate_test.py
+++ b/integration_tests/src/main/python/hash_aggregate_test.py
@@ -372,7 +372,6 @@ def test_computation_in_grpby_columns():
 @pytest.mark.parametrize('data_gen', _init_list_with_decimalbig, ids=idfn)
 @pytest.mark.parametrize('conf', get_params(_confs, params_markers_for_confs), ids=idfn)
 def test_hash_grpby_sum(data_gen, conf):
-    conf = copy_and_update(conf, {'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark: gen_df(spark, data_gen, length=100).groupby('a').agg(f.sum('b')),
         conf = conf)
@@ -385,7 +384,6 @@ def test_hash_grpby_sum(data_gen, conf):
 @pytest.mark.parametrize('data_gen', [_grpkey_short_sum_full_decimals, _grpkey_short_sum_full_neg_scale_decimals], ids=idfn)
 @pytest.mark.parametrize('conf', get_params(_confs, params_markers_for_confs), ids=idfn)
 def test_hash_grpby_sum_full_decimal(data_gen, conf):
-    conf = copy_and_update(conf, {'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark: gen_df(spark, data_gen, length=100).groupby('a').agg(f.sum('b')),
         conf = conf)
@@ -410,7 +408,6 @@ def test_hash_reduction_sum(data_gen, conf):
 @pytest.mark.parametrize('conf', get_params(_confs, params_markers_for_confs), ids=idfn)
 @datagen_overrides(seed=0, permanent=True, reason='https://github.com/NVIDIA/spark-rapids/issues/9779')
 def test_hash_reduction_sum_full_decimal(data_gen, conf):
-    conf = copy_and_update(conf, {'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark: unary_op_df(spark, data_gen, length=100).selectExpr("SUM(a)"),
         conf = conf)
@@ -455,8 +452,7 @@ def test_hash_avg_nulls_partial_only(data_gen):
 @pytest.mark.parametrize('data_gen', _init_list_with_decimalbig, ids=idfn)
 def test_intersect_all(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : gen_df(spark, data_gen, length=100).intersectAll(gen_df(spark, data_gen, length=100)),
-    conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+        lambda spark : gen_df(spark, data_gen, length=100).intersectAll(gen_df(spark, data_gen, length=100)))
 
 @approximate_float
 @ignore_order
@@ -465,8 +461,7 @@ def test_intersect_all(data_gen):
 @pytest.mark.parametrize('data_gen', _init_list_with_decimalbig, ids=idfn)
 def test_exceptAll(data_gen):
     assert_gpu_and_cpu_are_equal_collect(
-        lambda spark : gen_df(spark, data_gen, length=100).exceptAll(gen_df(spark, data_gen, length=100).filter('a != b')),
-        conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
+        lambda spark : gen_df(spark, data_gen, length=100).exceptAll(gen_df(spark, data_gen, length=100).filter('a != b')))
 
 # Spark fails to sort some decimal values due to overflow when calculating the sorting prefix.
 # See https://issues.apache.org/jira/browse/SPARK-40129
@@ -494,7 +489,6 @@ def test_exceptAll(data_gen):
 @pytest.mark.parametrize('data_gen', _pivot_gens_with_decimals, ids=idfn)
 @pytest.mark.parametrize('conf', get_params(_confs, params_markers_for_confs), ids=idfn)
 def test_hash_grpby_pivot(data_gen, conf):
-    conf = copy_and_update(conf, {'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'})
     assert_gpu_and_cpu_are_equal_collect(
         lambda spark: gen_df(spark, data_gen, length=100)
             .groupby('a')
@@ -1116,8 +1110,7 @@ def test_hash_groupby_typed_imperative_agg_without_gpu_implementation_fallback()
         non_exist_classes='GpuApproximatePercentile,GpuObjectHashAggregateExec',
         table_name='table',
         sql="""select k,
-        approx_percentile(v, array(0.25, 0.5, 0.75)) from table group by k""",
-    conf={'spark.rapids.sql.incompatibleOps.enabled': 'false'})
+        approx_percentile(v, array(0.25, 0.5, 0.75)) from table group by k""")
 
 @approximate_float
 @ignore_order
diff --git a/integration_tests/src/main/python/spark_session.py b/integration_tests/src/main/python/spark_session.py
index eeb6eb1c5e2..831680e4feb 100644
--- a/integration_tests/src/main/python/spark_session.py
+++ b/integration_tests/src/main/python/spark_session.py
@@ -16,12 +16,11 @@
 import calendar, time
 from datetime import date, datetime
 from contextlib import contextmanager, ExitStack
-from conftest import is_allowing_any_non_gpu, get_non_gpu_allowed, get_validate_execs_in_gpu_plan, is_databricks_runtime, is_at_least_precommit_run, get_inject_oom_conf, is_per_test_ansi_mode_enabled, get_default_configs_path
+from conftest import is_allowing_any_non_gpu, get_non_gpu_allowed, get_validate_execs_in_gpu_plan, is_databricks_runtime, is_at_least_precommit_run, get_inject_oom_conf, is_per_test_ansi_mode_enabled
 from pyspark.sql import DataFrame
 from pyspark.sql.types import TimestampType, DateType, _acceptable_types
 from spark_init_internal import get_spark_i_know_what_i_am_doing, spark_version
 from unittest.mock import patch
-import json
 
 def _from_scala_map(scala_map):
     ret = {}
@@ -41,8 +40,23 @@ def _from_scala_map(scala_map):
 # These settings can be overridden by specific tests if necessary.
 # Many of these are redundant with default settings for the configs but are set here explicitly
 # to ensure any cluster settings do not interfere with tests that assume the defaults.
-with open(get_default_configs_path()) as conf_file:
-    _default_conf = json.load(conf_file)
+_default_conf = {
+    'spark.rapids.sql.castDecimalToFloat.enabled': 'false',
+    'spark.rapids.sql.castFloatToDecimal.enabled': 'false',
+    'spark.rapids.sql.castFloatToIntegralTypes.enabled': 'false',
+    'spark.rapids.sql.castFloatToString.enabled': 'false',
+    'spark.rapids.sql.castStringToFloat.enabled': 'false',
+    'spark.rapids.sql.castStringToTimestamp.enabled': 'false',
+    'spark.rapids.sql.fast.sample': 'false',
+    'spark.rapids.sql.hasExtendedYearValues': 'true',
+    'spark.rapids.sql.hashOptimizeSort.enabled': 'false',
+    'spark.rapids.sql.improvedFloatOps.enabled': 'false',
+    'spark.rapids.sql.incompatibleDateFormats.enabled': 'false',
+    'spark.rapids.sql.incompatibleOps.enabled': 'false',
+    'spark.rapids.sql.mode': 'executeongpu',
+    'spark.rapids.sql.variableFloatAgg.enabled': 'false',
+    'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true',
+}
 
 def _set_all_confs(conf):
     newconf = _default_conf.copy()
@@ -53,10 +67,7 @@ def _set_all_confs(conf):
     newconf.update(conf)
     for key, value in newconf.items():
         if _spark.conf.get(key, None) != value:
-            if isinstance(value, str):
-                _spark.conf.set(key, value)
-            else:
-                _spark.conf.set(key, str(value))
+            _spark.conf.set(key, value)
 
 def reset_spark_session_conf():
     """Reset all of the configs for a given spark session."""
diff --git a/integration_tests/src/main/python/string_test.py b/integration_tests/src/main/python/string_test.py
index e9bc9db6462..4ae4a827aa0 100644
--- a/integration_tests/src/main/python/string_test.py
+++ b/integration_tests/src/main/python/string_test.py
@@ -878,8 +878,7 @@ def test_format_number_supported(data_gen):
             'format_number(a, 10)',
             'format_number(a, 13)',
             'format_number(a, 30)',
-            'format_number(a, 100)'),
-        conf={'spark.sql.legacy.allowNegativeScaleOfDecimal': 'true'}
+            'format_number(a, 100)')
     )
 
 format_float_special_vals = [float('nan'), float('inf'), float('-inf'), 0.0, -0.0, 
diff --git a/integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/DumpDefaultConfigs.scala b/integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/DumpDefaultConfigs.scala
deleted file mode 100644
index d3328cd42c3..00000000000
--- a/integration_tests/src/main/scala/com/nvidia/spark/rapids/tests/DumpDefaultConfigs.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2024, NVIDIA CORPORATION.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.nvidia.spark.rapids.tests
-
-import com.nvidia.spark.rapids.ShimReflectionUtils
-import org.apache.commons.lang3.reflect.MethodUtils
-
-/**
- * Dump all spark-rapids configs with their defaults.
- */
-object DumpDefaultConfigs {
-  def main(args: Array[String]): Unit = {
-    if (args.length != 2) {
-      System.err.println(s"Usage: ${this.getClass.getCanonicalName} {format} {output_path}")
-      System.exit(1)
-    }
-    // We use the reflection as RapidsConf should be accessed via the shim layer.
-    val clazz = ShimReflectionUtils.loadClass("com.nvidia.spark.rapids.RapidsConf")
-    MethodUtils.invokeStaticMethod(clazz, "dumpConfigsWithDefault", args(0), args(1))
-  }
-}
diff --git a/pom.xml b/pom.xml
index 54046364651..358aa0c52b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -939,20 +939,6 @@
                 <artifactId>iceberg-core</artifactId>
                 <version>${iceberg.version}</version>
                 <scope>provided</scope>
-                <exclusions>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-annotations</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-core</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-databind</artifactId>
-                    </exclusion>
-                </exclusions>
             </dependency>
             <dependency>
               <groupId>org.apache.spark</groupId>
@@ -1453,7 +1439,7 @@ This will force full Scala code rebuild in downstream modules.
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>exec-maven-plugin</artifactId>
-                    <version>3.3.0</version>
+                    <version>3.0.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
diff --git a/scala2.13/integration_tests/pom.xml b/scala2.13/integration_tests/pom.xml
index 6380d0dabb3..4c3ea72f341 100644
--- a/scala2.13/integration_tests/pom.xml
+++ b/scala2.13/integration_tests/pom.xml
@@ -68,16 +68,6 @@
             <!-- for hive udf test cases -->
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-hive_${scala.binary.version}</artifactId>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-databind</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <!-- https://mvnrepository.com/artifact/com.github.scopt/scopt -->
         <dependency>
@@ -88,31 +78,6 @@
     </dependencies>
     <build>
         <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>populate-default-configs-for-testing</id>
-                        <phase>generate-test-resources</phase>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <configuration>
-                            <target>
-                                <taskdef resource="net/sf/antcontrib/antcontrib.properties"/>
-                                <java classname="com.nvidia.spark.rapids.tests.DumpDefaultConfigs"
-                                    failonerror="true"
-                                    classpathref="maven.compile.classpath"
-                                    fork="true">
-                                    <arg value="json"/>
-                                    <arg value="${project.build.directory}/spark-rapids-default-configs.json"/>
-                                </java>
-                            </target>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
             <plugin>
                 <artifactId>maven-assembly-plugin</artifactId>
                 <version>3.6.0</version>
@@ -216,7 +181,7 @@
                 <artifactId>exec-maven-plugin</artifactId>
                 <executions>
                     <execution>
-                        <id>run-pyspark-tests</id>
+                        <id>run pyspark tests</id>
                         <phase>verify</phase><!--run after packageing and collecting dependencies-->
                         <goals>
                             <goal>exec</goal>
diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml
index cc33e64bf66..1161334a4c0 100644
--- a/scala2.13/pom.xml
+++ b/scala2.13/pom.xml
@@ -939,20 +939,6 @@
                 <artifactId>iceberg-core</artifactId>
                 <version>${iceberg.version}</version>
                 <scope>provided</scope>
-                <exclusions>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-annotations</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-core</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-databind</artifactId>
-                    </exclusion>
-                </exclusions>
             </dependency>
             <dependency>
               <groupId>org.apache.spark</groupId>
@@ -1453,7 +1439,7 @@ This will force full Scala code rebuild in downstream modules.
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>exec-maven-plugin</artifactId>
-                    <version>3.3.0</version>
+                    <version>3.0.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
index 494ba34237e..14551471e66 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
@@ -15,18 +15,15 @@
  */
 package com.nvidia.spark.rapids
 
-import java.io.{BufferedOutputStream, DataOutputStream, File, FileOutputStream}
-import java.nio.charset.StandardCharsets
+import java.io.{File, FileOutputStream}
 import java.util
-import java.util.Locale
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.{HashMap, ListBuffer}
 
 import ai.rapids.cudf.Cuda
 import com.nvidia.spark.rapids.jni.RmmSpark.OomInjectionType
 import com.nvidia.spark.rapids.lore.{LoreId, OutputLoreId}
-import org.json4s.DefaultFormats
-import org.json4s.jackson.Serialization.writePretty
-import scala.collection.JavaConverters._
-import scala.collection.mutable.{HashMap, ListBuffer}
 
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
@@ -127,7 +124,6 @@ abstract class ConfEntry[T](val key: String, val converter: String => T, val doc
 
   def get(conf: Map[String, String]): T
   def get(conf: SQLConf): T
-  def getDefault(): T
   def help(asTable: Boolean = false): Unit
 
   override def toString: String = key
@@ -151,10 +147,6 @@ class ConfEntryWithDefault[T](key: String, converter: String => T, doc: String,
     }
   }
 
-  override def getDefault(): T = {
-    defaultValue
-  }
-
   override def help(asTable: Boolean = false): Unit = {
     if (!isInternal) {
       val startupOnlyStr = if (isStartupOnly) "Startup" else "Runtime"
@@ -190,10 +182,6 @@ class OptionalConfEntry[T](key: String, val rawConverter: String => T, doc: Stri
     }
   }
 
-  override def getDefault(): Option[T] = {
-    None
-  }
-
   override def help(asTable: Boolean = false): Unit = {
     if (!isInternal) {
       val startupOnlyStr = if (isStartupOnly) "Startup" else "Runtime"
@@ -2386,17 +2374,6 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression.
     println("-----|-----------------|-------------|---------------|------")
   }
 
-  /**
-   * Returns all spark-rapids configs with their default values.
-   * This function is used to dump default configs, so that they
-   * could be used by the integration test.
-   */
-  def getAllConfigsWithDefault: Map[String, Any] = {
-    val allConfs = registeredConfs.clone()
-    allConfs.append(RapidsPrivateUtil.getPrivateConfigs(): _*)
-    allConfs.map(e => e.key -> e.getDefault).toMap
-  }
-
   def help(asTable: Boolean = false): Unit = {
     helpCommon(asTable)
     helpAdvanced(asTable)
@@ -2546,49 +2523,6 @@ val SHUFFLE_COMPRESSION_LZ4_CHUNK_SIZE = conf("spark.rapids.shuffle.compression.
       }
     }
   }
-
-  object Format extends Enumeration {
-    type Format = Value
-    val PLAIN, JSON = Value
-  }
-
-  def dumpConfigsWithDefault(formatName: String, outputPath: String): Unit = {
-    import com.nvidia.spark.rapids.Arm._
-
-    val format = Format.withName(formatName.toUpperCase(Locale.US))
-
-    println(s"Dumping all spark-rapids configs and their defaults at ${outputPath}")
-
-    val allConfs = getAllConfigsWithDefault
-    withResource(new FileOutputStream(outputPath)) { fos =>
-      withResource(new BufferedOutputStream(fos)) { bos =>
-        format match {
-          case Format.PLAIN =>
-            withResource(new DataOutputStream(bos)) { dos =>
-              allConfs.foreach( { case (k, v) =>
-                val valStr = v match {
-                  case Some(optVal) => optVal.toString
-                  case None => ""
-                  case _ =>
-                    if (v == null) {
-                      ""
-                    } else {
-                      v.toString
-                    }
-                }
-                dos.writeUTF(s"'${k}': '${valStr}',")
-              })
-            }
-          case Format.JSON =>
-            implicit val formats: DefaultFormats.type = DefaultFormats
-            bos.write(writePretty(allConfs)
-              .getBytes(StandardCharsets.UTF_8))
-          case _ =>
-            System.err.println(s"Unknown format: ${format}")
-        }
-      }
-    }
-  }
 }
 
 class RapidsConf(conf: Map[String, String]) extends Logging {