Merge branch 'ESPARK-119' into 'spark_2.1'

[ESPARK-119] 兼容hive mapjoin阈值的设置兼容`hive.mapjoin.smalltable.filesize`和`spark.sql.autoBroadcastJoinThreshold` resolve apache#119 See merge request !90
cenyuhai · Nov 18, 2017 · d2d03ba · d2d03ba
2 parents 357b4ae + cc97c1f
commit d2d03ba
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 1 deletion.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -109,7 +109,7 @@ object SQLConf {
       "run, and file-based data source tables where the statistics are computed directly on " +
       "the files of data.")
     .longConf
-    .createWithDefault(10L * 1024 * 1024)
+    .createWithDefault(25L * 1024 * 1024)
 
   val LIMIT_SCALE_UP_FACTOR = SQLConfigBuilder("spark.sql.limit.scaleUpFactor")
     .internal()
@@ -724,6 +724,7 @@ object SQLConf {
   object HiveVars {
     val REDUCE_BYTES = "hive.exec.reducers.bytes.per.reducer"
     val HIVE_DEFAULT_FILEFORMAT = "hive.default.fileformat"
+    val MAPJOIN_THRESHOLD = "hive.mapjoin.smalltable.filesize"
   }
 }
 

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
@@ -154,6 +154,14 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm
       }
       (keyValueOutput, runFunc)
 
+    case Some((SQLConf.HiveVars.MAPJOIN_THRESHOLD, None)) =>
+      val runFunc = (sparkSession: SparkSession) => {
+        Seq(Row(
+          SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key,
+          sparkSession.sessionState.conf.autoBroadcastJoinThreshold.toString))
+      }
+      (keyValueOutput, runFunc)
+
     // Queries a single property.
     case Some((key, None)) =>
       val runFunc = (sparkSession: SparkSession) => {
@@ -209,6 +217,12 @@ case class ResetCommand(key: Option[String]) extends RunnableCommand with Loggin
       }
       runFunc
 
+    case Some(SQLConf.HiveVars.MAPJOIN_THRESHOLD) =>
+      val runFunc = (sparkSession: SparkSession) => {
+        sparkSession.sessionState.conf.unsetConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key)
+      }
+      runFunc
+
     case Some(key) =>
       val runFunc = (sparkSession: SparkSession) => {
         sparkSession.sessionState.conf.unsetConf(key)