diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala index 90287c2028467..e669165f4f2f8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala @@ -616,7 +616,7 @@ case class EnsureRequirements( private def createKeyGroupedShuffleSpec( partitioning: Partitioning, distribution: ClusteredDistribution): Option[KeyGroupedShuffleSpec] = { - def check(partitioning: KeyGroupedPartitioning): Option[KeyGroupedShuffleSpec] = { + def tryCreate(partitioning: KeyGroupedPartitioning): Option[KeyGroupedShuffleSpec] = { val attributes = partitioning.expressions.flatMap(_.collectLeaves()) val clustering = distribution.clustering @@ -636,11 +636,10 @@ case class EnsureRequirements( } partitioning match { - case p: KeyGroupedPartitioning => check(p) + case p: KeyGroupedPartitioning => tryCreate(p) case PartitioningCollection(partitionings) => val specs = partitionings.map(p => createKeyGroupedShuffleSpec(p, distribution)) - assert(specs.forall(_.isEmpty) || specs.forall(_.isDefined)) - specs.head + specs.filter(_.isDefined).map(_.get).headOption case _ => None } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala index 99d99fede8485..6a146bc887db4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/KeyGroupedPartitioningSuite.scala @@ -369,6 +369,28 @@ class KeyGroupedPartitioningSuite extends DistributionAndOrderingSuiteBase { checkAnswer(df.sort("res"), Seq(Row(10.0), Row(15.5), Row(41.0))) } + test("SPARK-49179: Fix v2 multi bucketed inner joins throw AssertionError") { + val cols = Array( + Column.create("id", LongType), + Column.create("name", StringType)) + val buckets = Array(bucket(8, "id")) + + withTable("t1", "t2", "t3") { + Seq("t1", "t2", "t3").foreach { t => + createTable(t, cols, buckets) + sql(s"INSERT INTO testcat.ns.$t VALUES (1, 'aa'), (2, 'bb'), (3, 'cc')") + } + val df = sql( + """ + |SELECT t1.id, t2.id, t3.name FROM testcat.ns.t1 + |JOIN testcat.ns.t2 ON t1.id = t2.id + |JOIN testcat.ns.t3 ON t1.id = t3.id + |""".stripMargin) + checkAnswer(df, Seq(Row(1, 1, "aa"), Row(2, 2, "bb"), Row(3, 3, "cc"))) + assert(collectShuffles(df.queryExecution.executedPlan).isEmpty) + } + } + test("partitioned join: join with two partition keys and matching & sorted partitions") { val items_partitions = Array(bucket(8, "id"), days("arrive_time")) createTable(items, itemsColumns, items_partitions)