Skip to content

Commit

Permalink
got 3.5 compiling using a shim
Browse files Browse the repository at this point in the history
  • Loading branch information
scottsand-db committed Mar 19, 2024
1 parent 9071d64 commit 82699e2
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 6 deletions.
2 changes: 1 addition & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ lazy val spark = (project in file("spark"))
"org.apache.spark" %% "spark-sql" % sparkVersion % "test" classifier "tests",
"org.apache.spark" %% "spark-hive" % sparkVersion % "test" classifier "tests",
),
Compile / unmanagedSourceDirectories := Seq((baseDirectory in Compile).value, "src", "shim-spark-3.5"),
Compile / unmanagedSourceDirectories ++= Seq((baseDirectory in Compile).value / "src" / "shim-spark-3.5"),
// For adding staged Spark RC versions, Ex:
// resolvers += "Apche Spark 3.5.0 (RC1) Staging" at "https://repository.apache.org/content/repositories/orgapachespark-1444/",
Compile / packageBin / mappings := (Compile / packageBin / mappings).value ++
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import org.apache.spark.sql.delta.commands.VacuumCommand.getDeltaTable
import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand}
import org.apache.spark.sql.types.StringType

import foo.bar.UnresolvedTableShim

/**
* The `vacuum` command implementation for Spark SQL. Example SQL:
* {{{
Expand Down Expand Up @@ -77,7 +79,8 @@ object VacuumTableCommand {
dryRun: Boolean): VacuumTableCommand = {
val child = UnresolvedDeltaPathOrIdentifier(path, table, "VACUUM")
val unresolvedInventoryTable = inventoryTable.map(rt =>
UnresolvedTableShim.createUnresolvedTable(rt.nameParts, "VACUUM", relationTypeMismatchHint = None))
UnresolvedTableShim.createUnresolvedTable(
rt.nameParts, "VACUUM", relationTypeMismatchHint = None))
VacuumTableCommand(child, horizonHours, unresolvedInventoryTable, inventoryQuery, dryRun)
}
}
22 changes: 20 additions & 2 deletions spark/src/shim-spark-3.5/UnresolvedTableShim.scala
Original file line number Diff line number Diff line change
@@ -1,10 +1,28 @@
/*
* Copyright (2021) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package foo.bar

import org.apache.spark.sql.catalyst.analysis.UnresolvedTable

object UnresolvedTableShim {
def createUnresolvedTable(
tableNameParts: Seq[String],
commandName: String,
relationTypeMismatchHint: Option[String] = None) = {
relationTypeMismatchHint: Option[String] = None): UnresolvedTable = {
UnresolvedTable(tableNameParts, commandName, relationTypeMismatchHint)
}
}
}
20 changes: 18 additions & 2 deletions spark/src/shim-spark-4.0/UnresolvedTableShim.scala
Original file line number Diff line number Diff line change
@@ -1,10 +1,26 @@
/*
* Copyright (2021) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import org.apache.spark.sql.catalyst.analysis.UnresolvedTable

object UnresolvedTableShim {
def createUnresolvedTable(
tableNameParts: Seq[String],
commandName: String,
relationTypeMismatchHint: Option[String] = None) = {
relationTypeMismatchHint: Option[String] = None): UnresolvedTable = {
UnresolvedTable(tableNameParts, commandName)
}
}
}

0 comments on commit 82699e2

Please sign in to comment.