diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala new file mode 100644 index 0000000000000..08291859a32cc --- /dev/null +++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala @@ -0,0 +1,658 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +import java.io.{File, IOException, PrintStream} +import java.net.URI +import java.text.ParseException +import java.util.UUID + +import org.apache.commons.lang3.StringUtils +import org.apache.ivy.Ivy +import org.apache.ivy.core.LogOptions +import org.apache.ivy.core.module.descriptor.{Artifact, DefaultDependencyDescriptor, DefaultExcludeRule, DefaultModuleDescriptor, ExcludeRule} +import org.apache.ivy.core.module.id.{ArtifactId, ModuleId, ModuleRevisionId} +import org.apache.ivy.core.report.{DownloadStatus, ResolveReport} +import org.apache.ivy.core.resolve.ResolveOptions +import org.apache.ivy.core.retrieve.RetrieveOptions +import org.apache.ivy.core.settings.IvySettings +import org.apache.ivy.plugins.matcher.GlobPatternMatcher +import org.apache.ivy.plugins.repository.file.FileRepository +import org.apache.ivy.plugins.resolver.{ChainResolver, FileSystemResolver, IBiblioResolver} + +import org.apache.spark.SparkException +import org.apache.spark.internal.Logging +import org.apache.spark.util.ArrayImplicits._ + +/** Provides utility functions to be used inside SparkSubmit. */ +private[spark] object MavenUtils extends Logging { + val JAR_IVY_SETTING_PATH_KEY: String = "spark.jars.ivySettings" + + // Exposed for testing + // var printStream = SparkSubmit.printStream + + // Exposed for testing. + // These components are used to make the default exclusion rules for Spark dependencies. + // We need to specify each component explicitly, otherwise we miss + // spark-streaming utility components. Underscore is there to differentiate between + // spark-streaming_2.1x and spark-streaming-kafka-0-10-assembly_2.1x + val IVY_DEFAULT_EXCLUDES: Seq[String] = Seq( + "catalyst_", + "core_", + "graphx_", + "kvstore_", + "launcher_", + "mllib_", + "mllib-local_", + "network-common_", + "network-shuffle_", + "repl_", + "sketch_", + "sql_", + "streaming_", + "tags_", + "unsafe_") + + /** + * Represents a Maven Coordinate + * + * @param groupId + * the groupId of the coordinate + * @param artifactId + * the artifactId of the coordinate + * @param version + * the version of the coordinate + */ + private[spark] case class MavenCoordinate( + groupId: String, + artifactId: String, + version: String) { + override def toString: String = s"$groupId:$artifactId:$version" + } + + /** + * Extracts maven coordinates from a comma-delimited string. Coordinates should be provided in + * the format `groupId:artifactId:version` or `groupId/artifactId:version`. + * + * @param coordinates + * Comma-delimited string of maven coordinates + * @return + * Sequence of Maven coordinates + */ + def extractMavenCoordinates(coordinates: String): Seq[MavenCoordinate] = { + coordinates.split(",").map { p => + val splits = p.replace("/", ":").split(":") + require( + splits.length == 3, + s"Provided Maven Coordinates must be in the form " + + s"'groupId:artifactId:version'. The coordinate provided is: $p") + require( + splits(0) != null && splits(0).trim.nonEmpty, + s"The groupId cannot be null or " + + s"be whitespace. The groupId provided is: ${splits(0)}") + require( + splits(1) != null && splits(1).trim.nonEmpty, + s"The artifactId cannot be null or " + + s"be whitespace. The artifactId provided is: ${splits(1)}") + require( + splits(2) != null && splits(2).trim.nonEmpty, + s"The version cannot be null or " + + s"be whitespace. The version provided is: ${splits(2)}") + MavenCoordinate(splits(0), splits(1), splits(2)) + }.toImmutableArraySeq + } + + /** Path of the local Maven cache. */ + private[util] def m2Path: File = { + if (SparkEnvUtils.isTesting) { + // test builds delete the maven cache, and this can cause flakiness + new File("dummy", ".m2" + File.separator + "repository") + } else { + new File(System.getProperty("user.home"), ".m2" + File.separator + "repository") + } + } + + /** + * Create a ChainResolver used by Ivy to search for and resolve dependencies. + * + * @param defaultIvyUserDir + * The default user path for Ivy + * @param useLocalM2AsCache + * Whether to use the local maven repo as a cache + * @return + * A ChainResolver used by Ivy to search for and resolve dependencies. + */ + private[util] def createRepoResolvers( + defaultIvyUserDir: File, + useLocalM2AsCache: Boolean = true): ChainResolver = { + // We need a chain resolver if we want to check multiple repositories + val cr = new ChainResolver + cr.setName("spark-list") + + if (useLocalM2AsCache) { + val localM2 = new IBiblioResolver + localM2.setM2compatible(true) + localM2.setRoot(m2Path.toURI.toString) + localM2.setUsepoms(true) + localM2.setName("local-m2-cache") + cr.add(localM2) + } + + val localIvy = new FileSystemResolver + val localIvyRoot = new File(defaultIvyUserDir, "local") + localIvy.setLocal(true) + localIvy.setRepository(new FileRepository(localIvyRoot)) + val ivyPattern = Seq( + localIvyRoot.getAbsolutePath, + "[organisation]", + "[module]", + "[revision]", + "ivys", + "ivy.xml").mkString(File.separator) + localIvy.addIvyPattern(ivyPattern) + val artifactPattern = Seq( + localIvyRoot.getAbsolutePath, + "[organisation]", + "[module]", + "[revision]", + "[type]s", + "[artifact](-[classifier]).[ext]").mkString(File.separator) + localIvy.addArtifactPattern(artifactPattern) + localIvy.setName("local-ivy-cache") + cr.add(localIvy) + + // the biblio resolver resolves POM declared dependencies + val br: IBiblioResolver = new IBiblioResolver + br.setM2compatible(true) + br.setUsepoms(true) + val defaultInternalRepo: Option[String] = sys.env.get("DEFAULT_ARTIFACT_REPOSITORY") + br.setRoot(defaultInternalRepo.getOrElse("https://repo1.maven.org/maven2/")) + br.setName("central") + cr.add(br) + + val sp: IBiblioResolver = new IBiblioResolver + sp.setM2compatible(true) + sp.setUsepoms(true) + sp.setRoot( + sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", "https://repos.spark-packages.org/")) + sp.setName("spark-packages") + cr.add(sp) + cr + } + + /** + * Output a list of paths for the downloaded jars to be added to the classpath (will append to + * jars in SparkSubmit). + * + * @param artifacts + * Sequence of dependencies that were resolved and retrieved + * @param cacheDirectory + * Directory where jars are cached + * @return + * List of paths for the dependencies + */ + private[util] def resolveDependencyPaths( + artifacts: Array[AnyRef], + cacheDirectory: File): Seq[String] = { + artifacts + .map(_.asInstanceOf[Artifact]) + .filter { artifactInfo => + if (artifactInfo.getExt == "jar") { + true + } else { + logInfo(s"Skipping non-jar dependency ${artifactInfo.getId}") + false + } + } + .map { artifactInfo => + val artifact = artifactInfo.getModuleRevisionId + val extraAttrs = artifactInfo.getExtraAttributes + val classifier = if (extraAttrs.containsKey("classifier")) { + "-" + extraAttrs.get("classifier") + } else { + "" + } + cacheDirectory.getAbsolutePath + File.separator + + s"${artifact.getOrganisation}_${artifact.getName}-${artifact.getRevision}$classifier.jar" + }.toImmutableArraySeq + } + + /** Adds the given maven coordinates to Ivy's module descriptor. */ + private[util] def addDependenciesToIvy( + md: DefaultModuleDescriptor, + artifacts: Seq[MavenCoordinate], + ivyConfName: String)(implicit printStream: PrintStream): Unit = { + artifacts.foreach { mvn => + val ri = ModuleRevisionId.newInstance(mvn.groupId, mvn.artifactId, mvn.version) + val dd = new DefaultDependencyDescriptor(ri, false, false) + dd.addDependencyConfiguration(ivyConfName, ivyConfName + "(runtime)") + // scalastyle:off println + printStream.println(s"${dd.getDependencyId} added as a dependency") + // scalastyle:on println + md.addDependency(dd) + } + } + + /** Add exclusion rules for dependencies already included in the spark-assembly */ + private def addExclusionRules( + ivySettings: IvySettings, + ivyConfName: String, + md: DefaultModuleDescriptor): Unit = { + // Add scala exclusion rule + md.addExcludeRule(createExclusion("*:scala-library:*", ivySettings, ivyConfName)) + + IVY_DEFAULT_EXCLUDES.foreach { comp => + md.addExcludeRule( + createExclusion(s"org.apache.spark:spark-$comp*:*", ivySettings, ivyConfName)) + } + } + + /** + * Build Ivy Settings using options with default resolvers + * + * @param remoteRepos + * Comma-delimited string of remote repositories other than maven central + * @param ivyPath + * The path to the local ivy repository + * @param useLocalM2AsCache + * Whether or not use `local-m2 repo` as cache + * @return + * An IvySettings object + */ + def buildIvySettings( + remoteRepos: Option[String], + ivyPath: Option[String], + useLocalM2AsCache: Boolean = true)(implicit printStream: PrintStream): IvySettings = { + val ivySettings: IvySettings = new IvySettings + processIvyPathArg(ivySettings, ivyPath) + + // create a pattern matcher + ivySettings.addMatcher(new GlobPatternMatcher) + // create the dependency resolvers + val repoResolver = createRepoResolvers(ivySettings.getDefaultIvyUserDir, useLocalM2AsCache) + ivySettings.addResolver(repoResolver) + ivySettings.setDefaultResolver(repoResolver.getName) + processRemoteRepoArg(ivySettings, remoteRepos) + // (since 2.5) Setting the property ivy.maven.lookup.sources to false + // disables the lookup of the sources artifact. + // And setting the property ivy.maven.lookup.javadoc to false + // disables the lookup of the javadoc artifact. + ivySettings.setVariable("ivy.maven.lookup.sources", "false") + ivySettings.setVariable("ivy.maven.lookup.javadoc", "false") + ivySettings + } + + /** + * Load Ivy settings from a given filename, using supplied resolvers + * + * @param settingsFile + * Path to Ivy settings file + * @param remoteRepos + * Comma-delimited string of remote repositories other than maven central + * @param ivyPath + * The path to the local ivy repository + * @return + * An IvySettings object + */ + def loadIvySettings(settingsFile: String, remoteRepos: Option[String], ivyPath: Option[String])( + implicit printStream: PrintStream): IvySettings = { + val uri = new URI(settingsFile) + val file = Option(uri.getScheme).getOrElse("file") match { + case "file" => new File(uri.getPath) + case scheme => + throw new IllegalArgumentException( + s"Scheme $scheme not supported in " + + JAR_IVY_SETTING_PATH_KEY) + } + require(file.exists(), s"Ivy settings file $file does not exist") + require(file.isFile, s"Ivy settings file $file is not a normal file") + val ivySettings: IvySettings = new IvySettings + try { + ivySettings.load(file) + if (ivySettings.getDefaultIvyUserDir == null && ivySettings.getDefaultCache == null) { + // To protect old Ivy-based systems like old Spark from Apache Ivy 2.5.2's incompatibility. + // `processIvyPathArg` can overwrite these later. + val alternateIvyDir = System.getProperty("ivy.home", + System.getProperty("user.home") + File.separator + ".ivy2.5.2") + ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir)) + ivySettings.setDefaultCache(new File(alternateIvyDir, "cache")) + } + } catch { + case e @ (_: IOException | _: ParseException) => + throw new SparkException(s"Failed when loading Ivy settings from $settingsFile", e) + } + processIvyPathArg(ivySettings, ivyPath) + processRemoteRepoArg(ivySettings, remoteRepos) + ivySettings + } + + /* Set ivy settings for location of cache, if option is supplied */ + private def processIvyPathArg(ivySettings: IvySettings, ivyPath: Option[String]): Unit = { + val alternateIvyDir = ivyPath.filterNot(_.trim.isEmpty).getOrElse { + // To protect old Ivy-based systems like old Spark from Apache Ivy 2.5.2's incompatibility. + System.getProperty("ivy.home", + System.getProperty("user.home") + File.separator + ".ivy2.5.2") + } + ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir)) + ivySettings.setDefaultCache(new File(alternateIvyDir, "cache")) + } + + /* Add any optional additional remote repositories */ + private def processRemoteRepoArg(ivySettings: IvySettings, remoteRepos: Option[String])(implicit + printStream: PrintStream): Unit = { + remoteRepos.filterNot(_.trim.isEmpty).map(_.split(",")).foreach { repositoryList => + val cr = new ChainResolver + cr.setName("user-list") + + // add current default resolver, if any + Option(ivySettings.getDefaultResolver).foreach(cr.add) + + // add additional repositories, last resolution in chain takes precedence + repositoryList.zipWithIndex.foreach { case (repo, i) => + val brr: IBiblioResolver = new IBiblioResolver + brr.setM2compatible(true) + brr.setUsepoms(true) + brr.setRoot(repo) + brr.setName(s"repo-${i + 1}") + cr.add(brr) + // scalastyle:off println + printStream.println(s"$repo added as a remote repository with the name: ${brr.getName}") + // scalastyle:on println + } + + ivySettings.addResolver(cr) + ivySettings.setDefaultResolver(cr.getName) + } + } + + /** A nice function to use in tests as well. Values are dummy strings. */ + private[util] def getModuleDescriptor: DefaultModuleDescriptor = + DefaultModuleDescriptor.newDefaultInstance(ModuleRevisionId + // Include UUID in module name, so multiple clients resolving maven coordinate at the + // same time do not modify the same resolution file concurrently. + .newInstance("org.apache.spark", s"spark-submit-parent-${UUID.randomUUID.toString}", "1.0")) + + /** + * Clear ivy resolution from current launch. The resolution file is usually at + * ~/.ivy2/org.apache.spark-spark-submit-parent-$UUID-default.xml, + * ~/.ivy2/resolved-org.apache.spark-spark-submit-parent-$UUID-1.0.xml, and + * ~/.ivy2/resolved-org.apache.spark-spark-submit-parent-$UUID-1.0.properties. Since each launch + * will have its own resolution files created, delete them after each resolution to prevent + * accumulation of these files in the ivy cache dir. + */ + private def clearIvyResolutionFiles( + mdId: ModuleRevisionId, + defaultCacheFile: File, + ivyConfName: String): Unit = { + val currentResolutionFiles = Seq( + s"${mdId.getOrganisation}-${mdId.getName}-$ivyConfName.xml", + s"resolved-${mdId.getOrganisation}-${mdId.getName}-${mdId.getRevision}.xml", + s"resolved-${mdId.getOrganisation}-${mdId.getName}-${mdId.getRevision}.properties") + currentResolutionFiles.foreach { filename => + new File(defaultCacheFile, filename).delete() + } + } + + /** + * Clear invalid cache files in ivy. The cache file is usually at + * ~/.ivy2/cache/${groupId}/${artifactId}/ivy-${version}.xml, + * ~/.ivy2/cache/${groupId}/${artifactId}/ivy-${version}.xml.original, and + * ~/.ivy2/cache/${groupId}/${artifactId}/ivydata-${version}.properties. + * Because when using `local-m2` repo as a cache, some invalid files were created. + * If not deleted here, an error prompt similar to `unknown resolver local-m2-cache` + * will be generated, making some confusion for users. + */ + private def clearInvalidIvyCacheFiles( + mdId: ModuleRevisionId, + defaultCacheFile: File): Unit = { + val cacheFiles = Seq( + s"${mdId.getOrganisation}${File.separator}${mdId.getName}${File.separator}" + + s"ivy-${mdId.getRevision}.xml", + s"${mdId.getOrganisation}${File.separator}${mdId.getName}${File.separator}" + + s"ivy-${mdId.getRevision}.xml.original", + s"${mdId.getOrganisation}${File.separator}${mdId.getName}${File.separator}" + + s"ivydata-${mdId.getRevision}.properties") + cacheFiles.foreach { filename => + new File(defaultCacheFile, filename).delete() + } + } + + /** + * Resolves any dependencies that were supplied through maven coordinates + * + * @param coordinates + * Comma-delimited string of maven coordinates + * @param ivySettings + * An IvySettings containing resolvers to use + * @param noCacheIvySettings + * An no-cache(local-m2-cache) IvySettings containing resolvers to use + * @param transitive + * Whether resolving transitive dependencies, default is true + * @param exclusions + * Exclusions to apply when resolving transitive dependencies + * @return + * Seq of path to the jars of the given maven artifacts including their transitive + * dependencies + */ + def resolveMavenCoordinates( + coordinates: String, + ivySettings: IvySettings, + noCacheIvySettings: Option[IvySettings] = None, + transitive: Boolean, + exclusions: Seq[String] = Nil, + isTest: Boolean = false)(implicit printStream: PrintStream): Seq[String] = { + if (coordinates == null || coordinates.trim.isEmpty) { + Nil + } else { + val sysOut = System.out + // Default configuration name for ivy + val ivyConfName = "default" + + // A Module descriptor must be specified. Entries are dummy strings + val md = getModuleDescriptor + + md.setDefaultConf(ivyConfName) + try { + // To prevent ivy from logging to system out + System.setOut(printStream) + val artifacts = extractMavenCoordinates(coordinates) + // Directories for caching downloads through ivy and storing the jars when maven coordinates + // are supplied to spark-submit + val packagesDirectory: File = new File(ivySettings.getDefaultIvyUserDir, "jars") + // scalastyle:off println + printStream.println( + s"Ivy Default Cache set to: ${ivySettings.getDefaultCache.getAbsolutePath}") + printStream.println(s"The jars for the packages stored in: $packagesDirectory") + // scalastyle:on println + + val ivy = Ivy.newInstance(ivySettings) + ivy.pushContext() + + // Set resolve options to download transitive dependencies as well + val resolveOptions = new ResolveOptions + resolveOptions.setTransitive(transitive) + val retrieveOptions = new RetrieveOptions + // Turn downloading and logging off for testing + if (isTest) { + resolveOptions.setDownload(false) + resolveOptions.setLog(LogOptions.LOG_QUIET) + retrieveOptions.setLog(LogOptions.LOG_QUIET) + } else { + resolveOptions.setDownload(true) + } + // retrieve all resolved dependencies + retrieveOptions.setDestArtifactPattern( + packagesDirectory.getAbsolutePath + File.separator + + "[organization]_[artifact]-[revision](-[classifier]).[ext]") + retrieveOptions.setConfs(Array(ivyConfName)) + + // Add exclusion rules for Spark and Scala Library + addExclusionRules(ivySettings, ivyConfName, md) + // add all supplied maven artifacts as dependencies + addDependenciesToIvy(md, artifacts, ivyConfName) + exclusions.foreach { e => + md.addExcludeRule(createExclusion(e + ":*", ivySettings, ivyConfName)) + } + // resolve dependencies + val rr: ResolveReport = ivy.resolve(md, resolveOptions) + if (rr.hasError) { + // SPARK-46302: When there are some corrupted jars in the local maven repo, + // we try to continue without the cache + val failedReports = rr.getArtifactsReports(DownloadStatus.FAILED, true) + if (failedReports.nonEmpty && noCacheIvySettings.isDefined) { + val failedArtifacts = failedReports.map(r => r.getArtifact) + logInfo(s"Download failed: ${failedArtifacts.mkString("[", ", ", "]")}, " + + s"attempt to retry while skipping local-m2-cache.") + failedArtifacts.foreach(artifact => { + clearInvalidIvyCacheFiles(artifact.getModuleRevisionId, ivySettings.getDefaultCache) + }) + ivy.popContext() + + val noCacheIvy = Ivy.newInstance(noCacheIvySettings.get) + noCacheIvy.pushContext() + + val noCacheRr = noCacheIvy.resolve(md, resolveOptions) + if (noCacheRr.hasError) { + throw new RuntimeException(noCacheRr.getAllProblemMessages.toString) + } + noCacheIvy.retrieve(noCacheRr.getModuleDescriptor.getModuleRevisionId, retrieveOptions) + val dependencyPaths = resolveDependencyPaths( + noCacheRr.getArtifacts.toArray, packagesDirectory) + noCacheIvy.popContext() + + dependencyPaths + } else { + throw new RuntimeException(rr.getAllProblemMessages.toString) + } + } else { + ivy.retrieve(rr.getModuleDescriptor.getModuleRevisionId, retrieveOptions) + val dependencyPaths = resolveDependencyPaths(rr.getArtifacts.toArray, packagesDirectory) + ivy.popContext() + + dependencyPaths + } + } finally { + System.setOut(sysOut) + clearIvyResolutionFiles(md.getModuleRevisionId, ivySettings.getDefaultCache, ivyConfName) + } + } + } + + private[util] def createExclusion( + coords: String, + ivySettings: IvySettings, + ivyConfName: String): ExcludeRule = { + val c = extractMavenCoordinates(coords).head + val id = new ArtifactId(new ModuleId(c.groupId, c.artifactId), "*", "*", "*") + val rule = new DefaultExcludeRule(id, ivySettings.getMatcher("glob"), null) + rule.addConfiguration(ivyConfName) + rule + } + + private def isInvalidQueryString(tokens: Array[String]): Boolean = { + tokens.length != 2 || StringUtils.isBlank(tokens(0)) || StringUtils.isBlank(tokens(1)) + } + + /** + * Parse URI query string's parameter value of `transitive`, `exclude` and `repos`. + * Other invalid parameters will be ignored. + * + * @param uri + * Ivy URI need to be downloaded. + * @return + * Tuple value of parameter `transitive`, `exclude` and `repos` value. + * + * 1. transitive: whether to download dependency jar of Ivy URI, default value is true and + * this parameter value is case-insensitive. This mimics Hive's behaviour for parsing the + * transitive parameter. Invalid value will be treat as false. Example: Input: + * exclude=org.mortbay.jetty:jetty&transitive=true Output: true + * + * 2. exclude: comma separated exclusions to apply when resolving transitive dependencies, + * consists of `group:module` pairs separated by commas. Example: Input: + * excludeorg.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http Output: + * [org.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http] + * + * 3. repos: comma separated repositories to use when resolving dependencies. + */ + def parseQueryParams(uri: URI): (Boolean, String, String) = { + val uriQuery = uri.getQuery + if (uriQuery == null) { + (true, "", "") + } else { + val mapTokens = uriQuery.split("&").map(_.split("=")) + if (mapTokens.exists(MavenUtils.isInvalidQueryString)) { + throw new IllegalArgumentException( + s"Invalid query string in Ivy URI ${uri.toString}: $uriQuery") + } + val groupedParams = mapTokens.map(kv => (kv(0), kv(1))).groupBy(_._1) + + // Parse transitive parameters (e.g., transitive=true) in an Ivy URI, default value is true + val transitiveParams = groupedParams.get("transitive") + if (transitiveParams.map(_.length).getOrElse(0) > 1) { + logWarning( + "It's best to specify `transitive` parameter in ivy URI query only once." + + " If there are multiple `transitive` parameter, we will select the last one") + } + val transitive = + transitiveParams + .flatMap(_.takeRight(1).map(_._2.equalsIgnoreCase("true")).headOption) + .getOrElse(true) + + // Parse an excluded list (e.g., exclude=org.mortbay.jetty:jetty,org.eclipse.jetty:jetty-http) + // in an Ivy URI. When download Ivy URI jar, Spark won't download transitive jar + // in a excluded list. + val exclusionList = groupedParams + .get("exclude") + .map { params => + params + .map(_._2) + .flatMap { excludeString => + val excludes = excludeString.split(",") + if (excludes.map(_.split(":")).exists(MavenUtils.isInvalidQueryString)) { + throw new IllegalArgumentException( + s"Invalid exclude string in Ivy URI ${uri.toString}:" + + " expected 'org:module,org:module,..', found " + excludeString) + } + excludes + } + .mkString(",") + } + .getOrElse("") + + val repos = groupedParams + .get("repos") + .map { params => + params + .map(_._2) + .flatMap(_.split(",")) + .mkString(",") + } + .getOrElse("") + + val validParams = Set("transitive", "exclude", "repos") + val invalidParams = groupedParams.keys.filterNot(validParams.contains).toSeq + if (invalidParams.nonEmpty) { + logWarning( + s"Invalid parameters `${invalidParams.sorted.mkString(",")}` found " + + s"in Ivy URI query `$uriQuery`.") + } + + (transitive, exclusionList, repos) + } + } +} diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 56f4110b1180b..a9eb7aef1313c 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -2193,10 +2193,10 @@ package object config { .doc("Path to specify the Ivy user directory, used for the local Ivy cache and " + "package files from spark.jars.packages. " + "This will override the Ivy property ivy.default.ivy.user.dir " + - "which defaults to ~/.ivy2.") + "which defaults to ~/.ivy2.5.2") .version("1.3.0") .stringConf - .createOptional + .createWithDefault("~/.ivy2.5.2") private[spark] val JAR_IVY_SETTING_PATH = ConfigBuilder("spark.jars.ivySettings") diff --git a/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala index b986be03e965c..ebed572a39ab9 100644 --- a/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala +++ b/core/src/test/scala/org/apache/spark/deploy/IvyTestUtils.scala @@ -377,7 +377,8 @@ private[deploy] object IvyTestUtils { f(repo.toURI.toString) } finally { // Clean up - if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2")) { + if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2") || + repo.toString.contains(".ivy2.5.2")) { val groupDir = getBaseGroupDirectory(artifact, useIvyLayout) FileUtils.deleteDirectory(new File(repo, groupDir + File.separator + artifact.artifactId)) deps.foreach { _.foreach { dep => diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 new file mode 100644 index 0000000000000..283d93a4e60a3 --- /dev/null +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -0,0 +1,274 @@ +HikariCP/2.5.1//HikariCP-2.5.1.jar +JLargeArrays/1.5//JLargeArrays-1.5.jar +JTransforms/3.1//JTransforms-3.1.jar +RoaringBitmap/0.9.35//RoaringBitmap-0.9.35.jar +ST4/4.0.4//ST4-4.0.4.jar +activation/1.1.1//activation-1.1.1.jar +aircompressor/0.21//aircompressor-0.21.jar +algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar +annotations/17.0.0//annotations-17.0.0.jar +antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar +antlr4-runtime/4.9.3//antlr4-runtime-4.9.3.jar +aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar +aopalliance/1.0//aopalliance-1.0.jar +apacheds-i18n/2.0.0-M15//apacheds-i18n-2.0.0-M15.jar +apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar +api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar +api-util/1.0.0-M20//api-util-1.0.0-M20.jar +arpack/3.0.2//arpack-3.0.2.jar +arpack_combined_all/0.1//arpack_combined_all-0.1.jar +arrow-format/10.0.0//arrow-format-10.0.0.jar +arrow-memory-core/10.0.0//arrow-memory-core-10.0.0.jar +arrow-memory-netty/10.0.0//arrow-memory-netty-10.0.0.jar +arrow-vector/10.0.0//arrow-vector-10.0.0.jar +audience-annotations/0.5.0//audience-annotations-0.5.0.jar +avro-ipc/1.11.1//avro-ipc-1.11.1.jar +avro-mapred/1.11.1//avro-mapred-1.11.1.jar +avro/1.11.1//avro-1.11.1.jar +azure-storage/2.0.0//azure-storage-2.0.0.jar +blas/3.0.2//blas-3.0.2.jar +bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar +breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar +breeze_2.12/2.1.0//breeze_2.12-2.1.0.jar +cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar +chill-java/0.10.0//chill-java-0.10.0.jar +chill_2.12/0.10.0//chill_2.12-0.10.0.jar +commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar +commons-cli/1.5.0//commons-cli-1.5.0.jar +commons-codec/1.15//commons-codec-1.15.jar +commons-collections/3.2.2//commons-collections-3.2.2.jar +commons-collections4/4.4//commons-collections4-4.4.jar +commons-compiler/3.1.7//commons-compiler-3.1.7.jar +commons-compress/1.21//commons-compress-1.21.jar +commons-configuration/1.6//commons-configuration-1.6.jar +commons-crypto/1.1.0//commons-crypto-1.1.0.jar +commons-dbcp/1.4//commons-dbcp-1.4.jar +commons-digester/1.8//commons-digester-1.8.jar +commons-httpclient/3.1//commons-httpclient-3.1.jar +commons-io/2.4//commons-io-2.4.jar +commons-lang/2.6//commons-lang-2.6.jar +commons-lang3/3.12.0//commons-lang3-3.12.0.jar +commons-logging/1.1.3//commons-logging-1.1.3.jar +commons-math3/3.6.1//commons-math3-3.6.1.jar +commons-net/3.1//commons-net-3.1.jar +commons-pool/1.5.4//commons-pool-1.5.4.jar +commons-text/1.10.0//commons-text-1.10.0.jar +compress-lzf/1.1//compress-lzf-1.1.jar +curator-client/2.7.1//curator-client-2.7.1.jar +curator-framework/2.7.1//curator-framework-2.7.1.jar +curator-recipes/2.7.1//curator-recipes-2.7.1.jar +datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar +datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar +datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar +derby/10.14.2.0//derby-10.14.2.0.jar +dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar +flatbuffers-java/1.12.0//flatbuffers-java-1.12.0.jar +gcs-connector/hadoop2-2.2.7/shaded/gcs-connector-hadoop2-2.2.7-shaded.jar +gmetric4j/1.0.10//gmetric4j-1.0.10.jar +gson/2.2.4//gson-2.2.4.jar +guava/14.0.1//guava-14.0.1.jar +guice-servlet/3.0//guice-servlet-3.0.jar +guice/3.0//guice-3.0.jar +hadoop-annotations/2.7.4//hadoop-annotations-2.7.4.jar +hadoop-auth/2.7.4//hadoop-auth-2.7.4.jar +hadoop-aws/2.7.4//hadoop-aws-2.7.4.jar +hadoop-azure/2.7.4//hadoop-azure-2.7.4.jar +hadoop-client/2.7.4//hadoop-client-2.7.4.jar +hadoop-common/2.7.4//hadoop-common-2.7.4.jar +hadoop-hdfs/2.7.4//hadoop-hdfs-2.7.4.jar +hadoop-mapreduce-client-app/2.7.4//hadoop-mapreduce-client-app-2.7.4.jar +hadoop-mapreduce-client-common/2.7.4//hadoop-mapreduce-client-common-2.7.4.jar +hadoop-mapreduce-client-core/2.7.4//hadoop-mapreduce-client-core-2.7.4.jar +hadoop-mapreduce-client-jobclient/2.7.4//hadoop-mapreduce-client-jobclient-2.7.4.jar +hadoop-mapreduce-client-shuffle/2.7.4//hadoop-mapreduce-client-shuffle-2.7.4.jar +hadoop-openstack/2.7.4//hadoop-openstack-2.7.4.jar +hadoop-yarn-api/2.7.4//hadoop-yarn-api-2.7.4.jar +hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar +hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar +hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar +hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar +hive-beeline/2.3.9//hive-beeline-2.3.9.jar +hive-cli/2.3.9//hive-cli-2.3.9.jar +hive-common/2.3.9//hive-common-2.3.9.jar +hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar +hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar +hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar +hive-metastore/2.3.9//hive-metastore-2.3.9.jar +hive-serde/2.3.9//hive-serde-2.3.9.jar +hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar +hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar +hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar +hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar +hive-shims/2.3.9//hive-shims-2.3.9.jar +hive-storage-api/2.7.3//hive-storage-api-2.7.3.jar +hive-vector-code-gen/2.3.9//hive-vector-code-gen-2.3.9.jar +hk2-api/2.6.1//hk2-api-2.6.1.jar +hk2-locator/2.6.1//hk2-locator-2.6.1.jar +hk2-utils/2.6.1//hk2-utils-2.6.1.jar +htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar +httpclient/4.5.13//httpclient-4.5.13.jar +httpcore/4.4.14//httpcore-4.4.14.jar +istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar +ivy/2.5.1//ivy-2.5.1.jar +jackson-annotations/2.14.0//jackson-annotations-2.14.0.jar +jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar +jackson-core/2.14.0//jackson-core-2.14.0.jar +jackson-databind/2.14.0//jackson-databind-2.14.0.jar +jackson-dataformat-cbor/2.14.0//jackson-dataformat-cbor-2.14.0.jar +jackson-dataformat-yaml/2.14.0//jackson-dataformat-yaml-2.14.0.jar +jackson-datatype-jsr310/2.14.0//jackson-datatype-jsr310-2.14.0.jar +jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar +jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar +jackson-module-scala_2.12/2.14.0//jackson-module-scala_2.12-2.14.0.jar +jackson-xc/1.9.13//jackson-xc-1.9.13.jar +jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar +jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar +jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar +jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar +jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar +jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar +janino/3.1.7//janino-3.1.7.jar +javassist/3.25.0-GA//javassist-3.25.0-GA.jar +javax.inject/1//javax.inject-1.jar +javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar +javolution/5.5.1//javolution-5.5.1.jar +jaxb-api/2.2.11//jaxb-api-2.2.11.jar +jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar +jcl-over-slf4j/2.0.3//jcl-over-slf4j-2.0.3.jar +jdo-api/3.0.1//jdo-api-3.0.1.jar +jersey-client/2.36//jersey-client-2.36.jar +jersey-common/2.36//jersey-common-2.36.jar +jersey-container-servlet-core/2.36//jersey-container-servlet-core-2.36.jar +jersey-container-servlet/2.36//jersey-container-servlet-2.36.jar +jersey-hk2/2.36//jersey-hk2-2.36.jar +jersey-server/2.36//jersey-server-2.36.jar +jetty-sslengine/6.1.26//jetty-sslengine-6.1.26.jar +jetty-util/6.1.26//jetty-util-6.1.26.jar +jetty-util/9.4.49.v20220914//jetty-util-9.4.49.v20220914.jar +jetty/6.1.26//jetty-6.1.26.jar +jline/2.14.6//jline-2.14.6.jar +joda-time/2.12.0//joda-time-2.12.0.jar +jodd-core/3.5.2//jodd-core-3.5.2.jar +jpam/1.1//jpam-1.1.jar +json/1.8//json-1.8.jar +json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar +json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar +json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar +json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar +jsp-api/2.1//jsp-api-2.1.jar +jsr305/3.0.0//jsr305-3.0.0.jar +jta/1.1//jta-1.1.jar +jul-to-slf4j/2.0.3//jul-to-slf4j-2.0.3.jar +kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar +kubernetes-client-api/6.2.0//kubernetes-client-api-6.2.0.jar +kubernetes-client/6.2.0//kubernetes-client-6.2.0.jar +kubernetes-httpclient-okhttp/6.2.0//kubernetes-httpclient-okhttp-6.2.0.jar +kubernetes-model-admissionregistration/6.2.0//kubernetes-model-admissionregistration-6.2.0.jar +kubernetes-model-apiextensions/6.2.0//kubernetes-model-apiextensions-6.2.0.jar +kubernetes-model-apps/6.2.0//kubernetes-model-apps-6.2.0.jar +kubernetes-model-autoscaling/6.2.0//kubernetes-model-autoscaling-6.2.0.jar +kubernetes-model-batch/6.2.0//kubernetes-model-batch-6.2.0.jar +kubernetes-model-certificates/6.2.0//kubernetes-model-certificates-6.2.0.jar +kubernetes-model-common/6.2.0//kubernetes-model-common-6.2.0.jar +kubernetes-model-coordination/6.2.0//kubernetes-model-coordination-6.2.0.jar +kubernetes-model-core/6.2.0//kubernetes-model-core-6.2.0.jar +kubernetes-model-discovery/6.2.0//kubernetes-model-discovery-6.2.0.jar +kubernetes-model-events/6.2.0//kubernetes-model-events-6.2.0.jar +kubernetes-model-extensions/6.2.0//kubernetes-model-extensions-6.2.0.jar +kubernetes-model-flowcontrol/6.2.0//kubernetes-model-flowcontrol-6.2.0.jar +kubernetes-model-gatewayapi/6.2.0//kubernetes-model-gatewayapi-6.2.0.jar +kubernetes-model-metrics/6.2.0//kubernetes-model-metrics-6.2.0.jar +kubernetes-model-networking/6.2.0//kubernetes-model-networking-6.2.0.jar +kubernetes-model-node/6.2.0//kubernetes-model-node-6.2.0.jar +kubernetes-model-policy/6.2.0//kubernetes-model-policy-6.2.0.jar +kubernetes-model-rbac/6.2.0//kubernetes-model-rbac-6.2.0.jar +kubernetes-model-scheduling/6.2.0//kubernetes-model-scheduling-6.2.0.jar +kubernetes-model-storageclass/6.2.0//kubernetes-model-storageclass-6.2.0.jar +lapack/3.0.2//lapack-3.0.2.jar +leveldbjni-all/1.8//leveldbjni-all-1.8.jar +libfb303/0.9.3//libfb303-0.9.3.jar +libthrift/0.12.0//libthrift-0.12.0.jar +log4j-1.2-api/2.19.0//log4j-1.2-api-2.19.0.jar +log4j-api/2.19.0//log4j-api-2.19.0.jar +log4j-core/2.19.0//log4j-core-2.19.0.jar +log4j-slf4j2-impl/2.19.0//log4j-slf4j2-impl-2.19.0.jar +logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar +lz4-java/1.8.0//lz4-java-1.8.0.jar +mesos/1.4.3/shaded-protobuf/mesos-1.4.3-shaded-protobuf.jar +metrics-core/4.2.12//metrics-core-4.2.12.jar +metrics-graphite/4.2.12//metrics-graphite-4.2.12.jar +metrics-jmx/4.2.12//metrics-jmx-4.2.12.jar +metrics-json/4.2.12//metrics-json-4.2.12.jar +metrics-jvm/4.2.12//metrics-jvm-4.2.12.jar +minlog/1.3.0//minlog-1.3.0.jar +netty-all/4.1.84.Final//netty-all-4.1.84.Final.jar +netty-buffer/4.1.84.Final//netty-buffer-4.1.84.Final.jar +netty-codec-http/4.1.84.Final//netty-codec-http-4.1.84.Final.jar +netty-codec-http2/4.1.84.Final//netty-codec-http2-4.1.84.Final.jar +netty-codec-socks/4.1.84.Final//netty-codec-socks-4.1.84.Final.jar +netty-codec/4.1.84.Final//netty-codec-4.1.84.Final.jar +netty-common/4.1.84.Final//netty-common-4.1.84.Final.jar +netty-handler-proxy/4.1.84.Final//netty-handler-proxy-4.1.84.Final.jar +netty-handler/4.1.84.Final//netty-handler-4.1.84.Final.jar +netty-resolver/4.1.84.Final//netty-resolver-4.1.84.Final.jar +netty-tcnative-classes/2.0.54.Final//netty-tcnative-classes-2.0.54.Final.jar +netty-transport-classes-epoll/4.1.84.Final//netty-transport-classes-epoll-4.1.84.Final.jar +netty-transport-classes-kqueue/4.1.84.Final//netty-transport-classes-kqueue-4.1.84.Final.jar +netty-transport-native-epoll/4.1.84.Final/linux-aarch_64/netty-transport-native-epoll-4.1.84.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.84.Final/linux-x86_64/netty-transport-native-epoll-4.1.84.Final-linux-x86_64.jar +netty-transport-native-kqueue/4.1.84.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.84.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.1.84.Final/osx-x86_64/netty-transport-native-kqueue-4.1.84.Final-osx-x86_64.jar +netty-transport-native-unix-common/4.1.84.Final//netty-transport-native-unix-common-4.1.84.Final.jar +netty-transport/4.1.84.Final//netty-transport-4.1.84.Final.jar +objenesis/3.2//objenesis-3.2.jar +okhttp/3.12.12//okhttp-3.12.12.jar +okio/1.15.0//okio-1.15.0.jar +opencsv/2.3//opencsv-2.3.jar +orc-core/1.8.0/shaded-protobuf/orc-core-1.8.0-shaded-protobuf.jar +orc-mapreduce/1.8.0/shaded-protobuf/orc-mapreduce-1.8.0-shaded-protobuf.jar +orc-shims/1.8.0//orc-shims-1.8.0.jar +oro/2.0.8//oro-2.0.8.jar +osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar +paranamer/2.8//paranamer-2.8.jar +parquet-column/1.12.3//parquet-column-1.12.3.jar +parquet-common/1.12.3//parquet-common-1.12.3.jar +parquet-encoding/1.12.3//parquet-encoding-1.12.3.jar +parquet-format-structures/1.12.3//parquet-format-structures-1.12.3.jar +parquet-hadoop/1.12.3//parquet-hadoop-1.12.3.jar +parquet-jackson/1.12.3//parquet-jackson-1.12.3.jar +pickle/1.2//pickle-1.2.jar +protobuf-java/2.5.0//protobuf-java-2.5.0.jar +py4j/0.10.9.7//py4j-0.10.9.7.jar +remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar +rocksdbjni/7.7.3//rocksdbjni-7.7.3.jar +scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar +scala-compiler/2.12.17//scala-compiler-2.12.17.jar +scala-library/2.12.17//scala-library-2.12.17.jar +scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar +scala-reflect/2.12.17//scala-reflect-2.12.17.jar +scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar +shims/0.9.35//shims-0.9.35.jar +slf4j-api/2.0.3//slf4j-api-2.0.3.jar +snakeyaml/1.33//snakeyaml-1.33.jar +snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar +spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar +spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar +spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar +spire_2.12/0.17.0//spire_2.12-0.17.0.jar +stax-api/1.0.1//stax-api-1.0.1.jar +stream/2.9.6//stream-2.9.6.jar +super-csv/2.2.0//super-csv-2.2.0.jar +threeten-extra/1.7.1//threeten-extra-1.7.1.jar +tink/1.7.0//tink-1.7.0.jar +transaction-api/1.1//transaction-api-1.1.jar +univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar +velocity/1.5//velocity-1.5.jar +xbean-asm9-shaded/4.22//xbean-asm9-shaded-4.22.jar +xercesImpl/2.12.2//xercesImpl-2.12.2.jar +xml-apis/1.4.01//xml-apis-1.4.01.jar +xmlenc/0.52//xmlenc-0.52.jar +xz/1.8//xz-1.8.jar +zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar +zookeeper-jute/3.6.2//zookeeper-jute-3.6.2.jar +zookeeper/3.6.2//zookeeper-3.6.2.jar +zstd-jni/1.5.2-5//zstd-jni-1.5.2-5.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 new file mode 100644 index 0000000000000..cd5b2159bff86 --- /dev/null +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -0,0 +1,259 @@ +HikariCP/2.5.1//HikariCP-2.5.1.jar +JLargeArrays/1.5//JLargeArrays-1.5.jar +JTransforms/3.1//JTransforms-3.1.jar +RoaringBitmap/0.9.45//RoaringBitmap-0.9.45.jar +ST4/4.0.4//ST4-4.0.4.jar +activation/1.1.1//activation-1.1.1.jar +aircompressor/0.25//aircompressor-0.25.jar +algebra_2.12/2.0.1//algebra_2.12-2.0.1.jar +aliyun-java-sdk-core/4.5.10//aliyun-java-sdk-core-4.5.10.jar +aliyun-java-sdk-kms/2.11.0//aliyun-java-sdk-kms-2.11.0.jar +aliyun-java-sdk-ram/3.1.0//aliyun-java-sdk-ram-3.1.0.jar +aliyun-sdk-oss/3.13.0//aliyun-sdk-oss-3.13.0.jar +annotations/17.0.0//annotations-17.0.0.jar +antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar +antlr4-runtime/4.9.3//antlr4-runtime-4.9.3.jar +aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar +arpack/3.0.3//arpack-3.0.3.jar +arpack_combined_all/0.1//arpack_combined_all-0.1.jar +arrow-format/12.0.1//arrow-format-12.0.1.jar +arrow-memory-core/12.0.1//arrow-memory-core-12.0.1.jar +arrow-memory-netty/12.0.1//arrow-memory-netty-12.0.1.jar +arrow-vector/12.0.1//arrow-vector-12.0.1.jar +audience-annotations/0.5.0//audience-annotations-0.5.0.jar +avro-ipc/1.11.2//avro-ipc-1.11.2.jar +avro-mapred/1.11.2//avro-mapred-1.11.2.jar +avro/1.11.2//avro-1.11.2.jar +aws-java-sdk-bundle/1.12.367//aws-java-sdk-bundle-1.12.367.jar +azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar +azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar +azure-storage/7.0.1//azure-storage-7.0.1.jar +blas/3.0.3//blas-3.0.3.jar +bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar +breeze-macros_2.12/2.1.0//breeze-macros_2.12-2.1.0.jar +breeze_2.12/2.1.0//breeze_2.12-2.1.0.jar +cats-kernel_2.12/2.1.1//cats-kernel_2.12-2.1.1.jar +chill-java/0.10.0//chill-java-0.10.0.jar +chill_2.12/0.10.0//chill_2.12-0.10.0.jar +commons-cli/1.5.0//commons-cli-1.5.0.jar +commons-codec/1.16.0//commons-codec-1.16.0.jar +commons-collections/3.2.2//commons-collections-3.2.2.jar +commons-collections4/4.4//commons-collections4-4.4.jar +commons-compiler/3.1.9//commons-compiler-3.1.9.jar +commons-compress/1.23.0//commons-compress-1.23.0.jar +commons-crypto/1.1.0//commons-crypto-1.1.0.jar +commons-dbcp/1.4//commons-dbcp-1.4.jar +commons-io/2.13.0//commons-io-2.13.0.jar +commons-lang/2.6//commons-lang-2.6.jar +commons-lang3/3.13.0//commons-lang3-3.13.0.jar +commons-logging/1.1.3//commons-logging-1.1.3.jar +commons-math3/3.6.1//commons-math3-3.6.1.jar +commons-pool/1.5.4//commons-pool-1.5.4.jar +commons-text/1.10.0//commons-text-1.10.0.jar +compress-lzf/1.1.2//compress-lzf-1.1.2.jar +curator-client/5.2.0//curator-client-5.2.0.jar +curator-framework/5.2.0//curator-framework-5.2.0.jar +curator-recipes/5.2.0//curator-recipes-5.2.0.jar +datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar +datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar +datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar +datasketches-java/3.3.0//datasketches-java-3.3.0.jar +datasketches-memory/2.1.0//datasketches-memory-2.1.0.jar +derby/10.14.2.0//derby-10.14.2.0.jar +dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar +flatbuffers-java/1.12.0//flatbuffers-java-1.12.0.jar +gcs-connector/hadoop3-2.2.17/shaded/gcs-connector-hadoop3-2.2.17-shaded.jar +gmetric4j/1.0.10//gmetric4j-1.0.10.jar +gson/2.2.4//gson-2.2.4.jar +guava/14.0.1//guava-14.0.1.jar +hadoop-aliyun/3.3.6//hadoop-aliyun-3.3.6.jar +hadoop-annotations/3.3.6//hadoop-annotations-3.3.6.jar +hadoop-aws/3.3.6//hadoop-aws-3.3.6.jar +hadoop-azure-datalake/3.3.6//hadoop-azure-datalake-3.3.6.jar +hadoop-azure/3.3.6//hadoop-azure-3.3.6.jar +hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar +hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar +hadoop-cloud-storage/3.3.6//hadoop-cloud-storage-3.3.6.jar +hadoop-shaded-guava/1.1.1//hadoop-shaded-guava-1.1.1.jar +hadoop-yarn-server-web-proxy/3.3.6//hadoop-yarn-server-web-proxy-3.3.6.jar +hive-beeline/2.3.9//hive-beeline-2.3.9.jar +hive-cli/2.3.9//hive-cli-2.3.9.jar +hive-common/2.3.9//hive-common-2.3.9.jar +hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar +hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar +hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar +hive-metastore/2.3.9//hive-metastore-2.3.9.jar +hive-serde/2.3.9//hive-serde-2.3.9.jar +hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar +hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar +hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar +hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar +hive-shims/2.3.9//hive-shims-2.3.9.jar +hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar +hk2-api/2.6.1//hk2-api-2.6.1.jar +hk2-locator/2.6.1//hk2-locator-2.6.1.jar +hk2-utils/2.6.1//hk2-utils-2.6.1.jar +httpclient/4.5.14//httpclient-4.5.14.jar +httpcore/4.4.16//httpcore-4.4.16.jar +ini4j/0.5.4//ini4j-0.5.4.jar +istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar +ivy/2.5.2//ivy-2.5.2.jar +jackson-annotations/2.16.1//jackson-annotations-2.16.1.jar +jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar +jackson-core/2.15.2//jackson-core-2.15.2.jar +jackson-databind/2.15.2//jackson-databind-2.15.2.jar +jackson-dataformat-cbor/2.15.2//jackson-dataformat-cbor-2.15.2.jar +jackson-dataformat-yaml/2.15.2//jackson-dataformat-yaml-2.15.2.jar +jackson-datatype-jsr310/2.15.2//jackson-datatype-jsr310-2.15.2.jar +jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar +jackson-module-scala_2.12/2.15.2//jackson-module-scala_2.12-2.15.2.jar +jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar +jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar +jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar +jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar +jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar +jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar +janino/3.1.9//janino-3.1.9.jar +javassist/3.29.2-GA//javassist-3.29.2-GA.jar +javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar +javolution/5.5.1//javolution-5.5.1.jar +jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar +jcl-over-slf4j/2.0.7//jcl-over-slf4j-2.0.7.jar +jdo-api/3.0.1//jdo-api-3.0.1.jar +jdom2/2.0.6//jdom2-2.0.6.jar +jersey-client/2.40//jersey-client-2.40.jar +jersey-common/2.40//jersey-common-2.40.jar +jersey-container-servlet-core/2.40//jersey-container-servlet-core-2.40.jar +jersey-container-servlet/2.40//jersey-container-servlet-2.40.jar +jersey-hk2/2.40//jersey-hk2-2.40.jar +jersey-server/2.40//jersey-server-2.40.jar +jettison/1.5.4//jettison-1.5.4.jar +jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar +jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar +jline/2.14.6//jline-2.14.6.jar +joda-time/2.12.5//joda-time-2.12.5.jar +jodd-core/3.5.2//jodd-core-3.5.2.jar +jpam/1.1//jpam-1.1.jar +json/1.8//json-1.8.jar +json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar +json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar +json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar +json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar +jsr305/3.0.0//jsr305-3.0.0.jar +jta/1.1//jta-1.1.jar +jul-to-slf4j/2.0.7//jul-to-slf4j-2.0.7.jar +kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar +kubernetes-client-api/6.8.1//kubernetes-client-api-6.8.1.jar +kubernetes-client/6.8.1//kubernetes-client-6.8.1.jar +kubernetes-httpclient-okhttp/6.8.1//kubernetes-httpclient-okhttp-6.8.1.jar +kubernetes-model-admissionregistration/6.8.1//kubernetes-model-admissionregistration-6.8.1.jar +kubernetes-model-apiextensions/6.8.1//kubernetes-model-apiextensions-6.8.1.jar +kubernetes-model-apps/6.8.1//kubernetes-model-apps-6.8.1.jar +kubernetes-model-autoscaling/6.8.1//kubernetes-model-autoscaling-6.8.1.jar +kubernetes-model-batch/6.8.1//kubernetes-model-batch-6.8.1.jar +kubernetes-model-certificates/6.8.1//kubernetes-model-certificates-6.8.1.jar +kubernetes-model-common/6.8.1//kubernetes-model-common-6.8.1.jar +kubernetes-model-coordination/6.8.1//kubernetes-model-coordination-6.8.1.jar +kubernetes-model-core/6.8.1//kubernetes-model-core-6.8.1.jar +kubernetes-model-discovery/6.8.1//kubernetes-model-discovery-6.8.1.jar +kubernetes-model-events/6.8.1//kubernetes-model-events-6.8.1.jar +kubernetes-model-extensions/6.8.1//kubernetes-model-extensions-6.8.1.jar +kubernetes-model-flowcontrol/6.8.1//kubernetes-model-flowcontrol-6.8.1.jar +kubernetes-model-gatewayapi/6.8.1//kubernetes-model-gatewayapi-6.8.1.jar +kubernetes-model-metrics/6.8.1//kubernetes-model-metrics-6.8.1.jar +kubernetes-model-networking/6.8.1//kubernetes-model-networking-6.8.1.jar +kubernetes-model-node/6.8.1//kubernetes-model-node-6.8.1.jar +kubernetes-model-policy/6.8.1//kubernetes-model-policy-6.8.1.jar +kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar +kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar +kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar +kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar +lapack/3.0.3//lapack-3.0.3.jar +leveldbjni-all/1.8//leveldbjni-all-1.8.jar +libfb303/0.9.3//libfb303-0.9.3.jar +libthrift/0.12.0//libthrift-0.12.0.jar +log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar +log4j-api/2.20.0//log4j-api-2.20.0.jar +log4j-core/2.20.0//log4j-core-2.20.0.jar +log4j-slf4j2-impl/2.20.0//log4j-slf4j2-impl-2.20.0.jar +logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar +lz4-java/1.8.0//lz4-java-1.8.0.jar +mesos/1.4.3/shaded-protobuf/mesos-1.4.3-shaded-protobuf.jar +metrics-core/4.2.19//metrics-core-4.2.19.jar +metrics-graphite/4.2.19//metrics-graphite-4.2.19.jar +metrics-jmx/4.2.19//metrics-jmx-4.2.19.jar +metrics-json/4.2.19//metrics-json-4.2.19.jar +metrics-jvm/4.2.19//metrics-jvm-4.2.19.jar +minlog/1.3.0//minlog-1.3.0.jar +netty-all/4.1.96.Final//netty-all-4.1.96.Final.jar +netty-buffer/4.1.96.Final//netty-buffer-4.1.96.Final.jar +netty-codec-http/4.1.96.Final//netty-codec-http-4.1.96.Final.jar +netty-codec-http2/4.1.96.Final//netty-codec-http2-4.1.96.Final.jar +netty-codec-socks/4.1.96.Final//netty-codec-socks-4.1.96.Final.jar +netty-codec/4.1.96.Final//netty-codec-4.1.96.Final.jar +netty-common/4.1.96.Final//netty-common-4.1.96.Final.jar +netty-handler-proxy/4.1.96.Final//netty-handler-proxy-4.1.96.Final.jar +netty-handler/4.1.96.Final//netty-handler-4.1.96.Final.jar +netty-resolver/4.1.96.Final//netty-resolver-4.1.96.Final.jar +netty-transport-classes-epoll/4.1.96.Final//netty-transport-classes-epoll-4.1.96.Final.jar +netty-transport-classes-kqueue/4.1.96.Final//netty-transport-classes-kqueue-4.1.96.Final.jar +netty-transport-native-epoll/4.1.96.Final/linux-aarch_64/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.96.Final/linux-x86_64/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar +netty-transport-native-kqueue/4.1.96.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.1.96.Final/osx-x86_64/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar +netty-transport-native-unix-common/4.1.96.Final//netty-transport-native-unix-common-4.1.96.Final.jar +netty-transport/4.1.96.Final//netty-transport-4.1.96.Final.jar +objenesis/3.3//objenesis-3.3.jar +okhttp/3.12.12//okhttp-3.12.12.jar +okio/1.15.0//okio-1.15.0.jar +opencsv/2.3//opencsv-2.3.jar +opentracing-api/0.33.0//opentracing-api-0.33.0.jar +opentracing-noop/0.33.0//opentracing-noop-0.33.0.jar +opentracing-util/0.33.0//opentracing-util-0.33.0.jar +orc-core/1.9.1/shaded-protobuf/orc-core-1.9.1-shaded-protobuf.jar +orc-mapreduce/1.9.1/shaded-protobuf/orc-mapreduce-1.9.1-shaded-protobuf.jar +orc-shims/1.9.1//orc-shims-1.9.1.jar +oro/2.0.8//oro-2.0.8.jar +osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar +paranamer/2.8//paranamer-2.8.jar +parquet-column/1.13.1//parquet-column-1.13.1.jar +parquet-common/1.13.1//parquet-common-1.13.1.jar +parquet-encoding/1.13.1//parquet-encoding-1.13.1.jar +parquet-format-structures/1.13.1//parquet-format-structures-1.13.1.jar +parquet-hadoop/1.13.1//parquet-hadoop-1.13.1.jar +parquet-jackson/1.13.1//parquet-jackson-1.13.1.jar +pickle/1.3//pickle-1.3.jar +py4j/0.10.9.7//py4j-0.10.9.7.jar +remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar +rocksdbjni/8.3.2//rocksdbjni-8.3.2.jar +scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar +scala-compiler/2.12.18//scala-compiler-2.12.18.jar +scala-library/2.12.18//scala-library-2.12.18.jar +scala-parser-combinators_2.12/2.3.0//scala-parser-combinators_2.12-2.3.0.jar +scala-reflect/2.12.18//scala-reflect-2.12.18.jar +scala-xml_2.12/2.2.0//scala-xml_2.12-2.2.0.jar +shims/0.9.45//shims-0.9.45.jar +slf4j-api/2.0.7//slf4j-api-2.0.7.jar +snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar +snakeyaml/2.0//snakeyaml-2.0.jar +snappy-java/1.1.10.3//snappy-java-1.1.10.3.jar +spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar +spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar +spire-util_2.12/0.17.0//spire-util_2.12-0.17.0.jar +spire_2.12/0.17.0//spire_2.12-0.17.0.jar +stax-api/1.0.1//stax-api-1.0.1.jar +stream/2.9.6//stream-2.9.6.jar +super-csv/2.2.0//super-csv-2.2.0.jar +threeten-extra/1.7.1//threeten-extra-1.7.1.jar +tink/1.9.0//tink-1.9.0.jar +transaction-api/1.1//transaction-api-1.1.jar +txw2/3.0.2//txw2-3.0.2.jar +univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar +wildfly-openssl/1.1.3.Final//wildfly-openssl-1.1.3.Final.jar +xbean-asm9-shaded/4.23//xbean-asm9-shaded-4.23.jar +xmlschema-core/2.3.0//xmlschema-core-2.3.0.jar +xz/1.9//xz-1.9.jar +zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar +zookeeper-jute/3.6.3//zookeeper-jute-3.6.3.jar +zookeeper/3.6.3//zookeeper-3.6.3.jar +zstd-jni/1.5.5-5//zstd-jni-1.5.5-5.jar diff --git a/dev/run-tests.py b/dev/run-tests.py index 3627571db4783..2ec44bd830b85 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -585,6 +585,8 @@ def main(): rm_r(os.path.join(SPARK_HOME, "work")) rm_r(os.path.join(USER_HOME, ".ivy2", "local", "org.apache.spark")) rm_r(os.path.join(USER_HOME, ".ivy2", "cache", "org.apache.spark")) + rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "local", "org.apache.spark")) + rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "cache", "org.apache.spark")) os.environ["CURRENT_BLOCK"] = str(ERROR_CODES["BLOCK_GENERAL"]) diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md index 3686fb3444c6e..3adfbeca8fd96 100644 --- a/docs/core-migration-guide.md +++ b/docs/core-migration-guide.md @@ -22,6 +22,42 @@ license: | * Table of contents {:toc} +## Upgrading from Core 3.5 to 4.0 + +- Since Spark 4.0, Spark will roll event logs to archive them incrementally. To restore the behavior before Spark 4.0, you can set `spark.eventLog.rolling.enabled` to `false`. + +- Since Spark 4.0, Spark will compress event logs. To restore the behavior before Spark 4.0, you can set `spark.eventLog.compress` to `false`. + +- Since Spark 4.0, Spark workers will clean up worker and stopped application directories periodically. To restore the behavior before Spark 4.0, you can set `spark.worker.cleanup.enabled` to `false`. + +- Since Spark 4.0, `spark.shuffle.service.db.backend` is set to `ROCKSDB` by default which means Spark will use RocksDB store for shuffle service. To restore the behavior before Spark 4.0, you can set `spark.shuffle.service.db.backend` to `LEVELDB`. + +- In Spark 4.0, support for Apache Mesos as a resource manager was removed. + +- Since Spark 4.0, Spark uses `ReadWriteOncePod` instead of `ReadWriteOnce` access mode in persistence volume claims. To restore the legacy behavior, you can set `spark.kubernetes.legacy.useReadWriteOnceAccessMode` to `true`. + +- Since Spark 4.0, Spark uses `~/.ivy2.5.2` as Ivy user directory by default to isolate the existing systems from Apache Ivy's incompatibility. To restore the legacy behavior, you can set `spark.jars.ivy` to `~/.ivy2`. + +## Upgrading from Core 3.4 to 3.5 + +- Since Spark 3.5, `spark.yarn.executor.failuresValidityInterval` is deprecated. Use `spark.executor.failuresValidityInterval` instead. + +- Since Spark 3.5, `spark.yarn.max.executor.failures` is deprecated. Use `spark.executor.maxNumFailures` instead. + +## Upgrading from Core 3.3 to 3.4 + +- Since Spark 3.4, Spark driver will own `PersistentVolumnClaim`s and try to reuse if they are not assigned to live executors. To restore the behavior before Spark 3.4, you can set `spark.kubernetes.driver.ownPersistentVolumeClaim` to `false` and `spark.kubernetes.driver.reusePersistentVolumeClaim` to `false`. + +- Since Spark 3.4, Spark driver will track shuffle data when dynamic allocation is enabled without shuffle service. To restore the behavior before Spark 3.4, you can set `spark.dynamicAllocation.shuffleTracking.enabled` to `false`. + +- Since Spark 3.4, Spark will try to decommission cached RDD and shuffle blocks if both `spark.decommission.enabled` and `spark.storage.decommission.enabled` are true. To restore the behavior before Spark 3.4, you can set both `spark.storage.decommission.rddBlocks.enabled` and `spark.storage.decommission.shuffleBlocks.enabled` to `false`. + +- Since Spark 3.4, Spark will use RocksDB store if `spark.history.store.hybridStore.enabled` is true. To restore the behavior before Spark 3.4, you can set `spark.history.store.hybridStore.diskBackend` to `LEVELDB`. + +## Upgrading from Core 3.2 to 3.3 + +- Since Spark 3.3, Spark migrates its log4j dependency from 1.x to 2.x because log4j 1.x has reached end of life and is no longer supported by the community. Vulnerabilities reported after August 2015 against log4j 1.x were not checked and will not be fixed. Users should rewrite original log4j properties files using log4j2 syntax (XML, JSON, YAML, or properties format). Spark rewrites the `conf/log4j.properties.template` which is included in Spark distribution, to `conf/log4j2.properties.template` with log4j2 properties format. + ## Upgrading from Core 3.1 to 3.2 - Since Spark 3.2, `spark.scheduler.allocation.file` supports read remote file using hadoop filesystem which means if the path has no scheme Spark will respect hadoop configuration to read it. To restore the behavior before Spark 3.2, you can specify the local scheme for `spark.scheduler.allocation.file` e.g. `file:///path/to/file`. diff --git a/pom.xml b/pom.xml index 05624002e3709..0b8b4c0ccdf52 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 9.4.44.v20210927 4.0.3 0.10.0 - 2.5.0 + 2.5.2 2.0.8