Add jobRunId to JobInfo

This PR adds a jobRunId field to JobInfo. When a multi-task job is created, the runId in JobInfo represents the task run ID, and jobRunId (retrieved from multitaskParentRunId in the CommandContext) represents the job run ID. GitOrigin-RevId: 1bd0027983de4af93e1e07e5237ac1c286853da8
delta-io · May 30, 2023 · df9d74d · df9d74d
1 parent 6eb78c2
commit df9d74d
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/core/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala b/core/src/main/scala/org/apache/spark/sql/delta/actions/actions.scala
@@ -1041,6 +1041,7 @@ case class CommitInfo(
 case class JobInfo(
     jobId: String,
     jobName: String,
+    jobRunId: String,
     runId: String,
     jobOwnerId: String,
     triggerType: String)
@@ -1051,6 +1052,7 @@ object JobInfo {
       JobInfo(
         jobId,
         context.get("jobName").orNull,
+        context.get("multitaskParentRunId").orNull,
         context.get("runId").orNull,
         context.get("jobOwnerId").orNull,
         context.get("jobTriggerType").orNull)

diff --git a/core/src/test/scala/org/apache/spark/sql/delta/DescribeDeltaHistorySuite.scala b/core/src/test/scala/org/apache/spark/sql/delta/DescribeDeltaHistorySuite.scala
@@ -1442,6 +1442,7 @@ trait DescribeDeltaHistorySuiteBase
           StructType(Seq(
             StructField("jobId", StringType, nullable = true),
             StructField("jobName", StringType, nullable = true),
+            StructField("jobRunId", StringType, nullable = true),
             StructField("runId", StringType, nullable = true),
             StructField("jobOwnerId", StringType, nullable = true),
             StructField("triggerType", StringType, nullable = true))),