Skip to content

Commit

Permalink
[SPARK-38018][SQL][3.2] Fix ColumnVectorUtils.populate to handle Cale…
Browse files Browse the repository at this point in the history
…ndarIntervalType correctly

### What changes were proposed in this pull request?

This is a backport of apache#35314 to branch 3.2. See that original PR for context.

### Why are the changes needed?

To fix potential correctness issue.

### Does this PR introduce _any_ user-facing change?

No but fix the exiting correctness issue when reading partition column with CalendarInterval type.

### How was this patch tested?

Added unit test in `ColumnVectorSuite.scala`.

Closes apache#37114 from c21/branch-3.2.

Authored-by: Cheng Su <scnju13@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
  • Loading branch information
c21 authored and cloud-fan committed Jul 7, 2022
1 parent 32aff86 commit c5983c1
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
} else if (t instanceof CalendarIntervalType) {
CalendarInterval c = (CalendarInterval)row.get(fieldIdx, t);
col.getChild(0).putInts(0, capacity, c.months);
col.getChild(1).putLongs(0, capacity, c.microseconds);
col.getChild(1).putInts(0, capacity, c.days);
col.getChild(2).putLongs(0, capacity, c.microseconds);
} else if (t instanceof DateType) {
col.putInts(0, capacity, row.getInt(fieldIdx));
} else if (t instanceof TimestampType || t instanceof TimestampNTZType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.columnar.ColumnAccessor
import org.apache.spark.sql.execution.columnar.compression.ColumnBuilderHelper
import org.apache.spark.sql.types._
import org.apache.spark.sql.vectorized.ColumnarArray
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}

class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
private def withVector(
Expand Down Expand Up @@ -536,5 +536,14 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}
}

test("SPARK-38018: ColumnVectorUtils.populate to handle CalendarIntervalType correctly") {
val vector = new OnHeapColumnVector(5, CalendarIntervalType)
val row = new SpecificInternalRow(Array(CalendarIntervalType))
val interval = new CalendarInterval(3, 5, 1000000)
row.setInterval(0, interval)
ColumnVectorUtils.populate(vector, row, 0)
assert(vector.getInterval(0) === interval)
}
}

0 comments on commit c5983c1

Please sign in to comment.