Reusing public function

itholic · Sep 4, 2024 · c8f9503 · c8f9503
1 parent ae98ed7
commit c8f9503
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 29 deletions.
diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/functions.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/functions.scala
@@ -71,7 +71,7 @@ object functions {
       messageName: String,
       binaryFileDescriptorSet: Array[Byte],
       options: java.util.Map[String, String]): Column = {
-    Column.internalFnWithOptions(
+    Column.fnWithOptions(
       "from_protobuf",
       options.asScala.iterator,
       data,
@@ -115,7 +115,7 @@ object functions {
   @Experimental
   def from_protobuf(data: Column, messageName: String, binaryFileDescriptorSet: Array[Byte])
   : Column = {
-    Column.internalFn(
+    Column.fn(
       "from_protobuf",
       data,
       lit(messageName),
@@ -140,7 +140,7 @@ object functions {
    */
   @Experimental
   def from_protobuf(data: Column, messageClassName: String): Column = {
-    Column.internalFn(
+    Column.fn(
       "from_protobuf",
       data,
       lit(messageClassName)
@@ -168,7 +168,7 @@ object functions {
     data: Column,
     messageClassName: String,
     options: java.util.Map[String, String]): Column = {
-    Column.internalFnWithOptions(
+    Column.fnWithOptions(
       "from_protobuf",
       options.asScala.iterator,
       data,
@@ -211,7 +211,7 @@ object functions {
   @Experimental
   def to_protobuf(data: Column, messageName: String, binaryFileDescriptorSet: Array[Byte])
   : Column = {
-    Column.internalFn(
+    Column.fn(
       "to_protobuf",
       data,
       lit(messageName),
@@ -262,7 +262,7 @@ object functions {
     binaryFileDescriptorSet: Array[Byte],
     options: java.util.Map[String, String]
   ): Column = {
-    Column.internalFnWithOptions(
+    Column.fnWithOptions(
       "to_protobuf",
       options.asScala.iterator,
       data,
@@ -288,7 +288,7 @@ object functions {
    */
   @Experimental
   def to_protobuf(data: Column, messageClassName: String): Column = {
-    Column.internalFn(
+    Column.fn(
       "to_protobuf",
       data,
       lit(messageClassName)
@@ -314,7 +314,7 @@ object functions {
   @Experimental
   def to_protobuf(data: Column, messageClassName: String, options: java.util.Map[String, String])
   : Column = {
-    Column.internalFnWithOptions(
+    Column.fnWithOptions(
       "to_protobuf",
       options.asScala.iterator,
       data,

diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala
@@ -67,25 +67,6 @@ private[spark] object Column {
     fn(name, isDistinct = false, isInternal = true, inputs)
   }
 
-  /**
-   * `fnWithOptions` corresponding for `internalFn`. If there are no options, its
-   * column is dropped.
-   */
-  private[sql] def internalFnWithOptions(
-      name: String,
-      options: Iterator[(String, String)],
-      arguments: Column*): Column = {
-    val augmentedArguments = if (options.hasNext) {
-      val flattenedKeyValueIterator = options.flatMap { case (k, v) =>
-        Iterator(lit(k), lit(v))
-      }
-      arguments :+ map(flattenedKeyValueIterator.toSeq: _*)
-    } else {
-      arguments
-    }
-    Column.internalFn(name, augmentedArguments: _*)
-  }
-
   private def fn(
       name: String,
       isDistinct: Boolean,

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -919,8 +919,6 @@ object FunctionRegistry {
   registerInternalExpression[EWM]("ewm")
   registerInternalExpression[NullIndex]("null_index")
   registerInternalExpression[CastTimestampNTZToLong]("timestamp_ntz_to_long")
-  registerInternalExpression[FromProtobuf]("from_protobuf")
-  registerInternalExpression[ToProtobuf]("to_protobuf")
   registerInternalExpression[ArrayBinarySearch]("array_binary_search")
 
   private def makeExprInfoForVirtualOperator(name: String, usage: String): ExpressionInfo = {