Added simpleName field to Job model and updated integration tests

Signed-off-by: Michael Collado <collado.mike@gmail.com>
MarquezProject · May 16, 2022 · 1f0e2fe · 1f0e2fe
1 parent 9ea2db6
commit 1f0e2fe
Show file tree

Hide file tree

Showing 11 changed files with 22 additions and 3 deletions.
diff --git a/api/src/main/java/marquez/db/mappers/JobDataMapper.java b/api/src/main/java/marquez/db/mappers/JobDataMapper.java
@@ -37,6 +37,7 @@ public JobData map(@NonNull ResultSet results, @NonNull StatementContext context
             JobName.of(stringOrThrow(results, Columns.NAME))),
         JobType.valueOf(stringOrThrow(results, Columns.TYPE)),
         JobName.of(stringOrThrow(results, Columns.NAME)),
+        stringOrThrow(results, Columns.SIMPLE_NAME),
         timestampOrThrow(results, Columns.CREATED_AT),
         timestampOrThrow(results, Columns.UPDATED_AT),
         NamespaceName.of(stringOrThrow(results, Columns.NAMESPACE_NAME)),

diff --git a/api/src/main/java/marquez/db/mappers/JobMapper.java b/api/src/main/java/marquez/db/mappers/JobMapper.java
@@ -44,6 +44,7 @@ public Job map(@NonNull ResultSet results, @NonNull StatementContext context)
             JobName.of(stringOrThrow(results, Columns.NAME))),
         JobType.valueOf(stringOrThrow(results, Columns.TYPE)),
         JobName.of(stringOrThrow(results, Columns.NAME)),
+        stringOrThrow(results, Columns.SIMPLE_NAME),
         timestampOrThrow(results, Columns.CREATED_AT),
         timestampOrThrow(results, Columns.UPDATED_AT),
         getDatasetFromJsonOrNull(results, "current_inputs"),

diff --git a/api/src/main/java/marquez/db/models/JobData.java b/api/src/main/java/marquez/db/models/JobData.java
@@ -31,6 +31,7 @@ public class JobData implements NodeData {
   @NonNull JobId id;
   @NonNull JobType type;
   @NonNull JobName name;
+  @NonNull String simpleName;
   @NonNull Instant createdAt;
   @NonNull Instant updatedAt;
   @NonNull NamespaceName namespace;

diff --git a/api/src/main/java/marquez/service/models/Job.java b/api/src/main/java/marquez/service/models/Job.java
@@ -26,6 +26,7 @@ public final class Job {
   @Getter private final JobId id;
   @Getter private final JobType type;
   @Getter private final JobName name;
+  @Getter private final String simpleName;
   @Getter private final Instant createdAt;
   @Getter private final Instant updatedAt;
   @Getter private final NamespaceName namespace;
@@ -42,6 +43,7 @@ public Job(
       @NonNull final JobId id,
       @NonNull final JobType type,
       @NonNull final JobName name,
+      @NonNull String simpleName,
       @NonNull final Instant createdAt,
       @NonNull final Instant updatedAt,
       @NonNull final Set<DatasetId> inputs,
@@ -55,6 +57,7 @@ public Job(
     this.id = id;
     this.type = type;
     this.name = name;
+    this.simpleName = simpleName;
     this.createdAt = createdAt;
     this.updatedAt = updatedAt;
     this.namespace = id.getNamespace();

diff --git a/api/src/test/java/marquez/OpenLineageIntegrationTest.java b/api/src/test/java/marquez/OpenLineageIntegrationTest.java
@@ -243,7 +243,8 @@ public void testOpenLineageJobHierarchyOldAirflowIntegration()
     Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name);
     assertThat(job)
         .isNotNull()
-        .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name));
+        .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name))
+        .hasFieldOrPropertyWithValue("simpleName", task1Name);
 
     Job parentJob = client.getJob(NAMESPACE_NAME, dagName);
     assertThat(parentJob)
@@ -288,13 +289,15 @@ public void testOpenLineageJobHierarchySparkAndAirflow()
     Job airflowTask = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name);
     assertThat(airflowTask)
         .isNotNull()
-        .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name));
+        .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name))
+        .hasFieldOrPropertyWithValue("simpleName", task1Name);
 
     Job sparkJob = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name + "." + sparkTaskName);
     assertThat(sparkJob)
         .isNotNull()
         .hasFieldOrPropertyWithValue(
-            "id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name + "." + sparkTaskName));
+            "id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name + "." + sparkTaskName))
+        .hasFieldOrPropertyWithValue("simpleName", sparkTaskName);
 
     Job parentJob = client.getJob(NAMESPACE_NAME, dagName);
     assertThat(parentJob)

diff --git a/api/src/test/java/marquez/db/mappers/JobMapperTest.java b/api/src/test/java/marquez/db/mappers/JobMapperTest.java
@@ -37,6 +37,8 @@ public static void setUp() throws SQLException, MalformedURLException {
     when(resultSet.getObject(Columns.NAMESPACE_NAME)).thenReturn("NAMESPACE");
     when(resultSet.getString(Columns.NAME)).thenReturn("NAME");
     when(resultSet.getObject(Columns.NAME)).thenReturn("NAME");
+    when(resultSet.getString(Columns.SIMPLE_NAME)).thenReturn("SIMPLE_NAME");
+    when(resultSet.getObject(Columns.SIMPLE_NAME)).thenReturn("SIMPLE_NAME");
     when(resultSet.getString(Columns.TYPE)).thenReturn("BATCH");
     when(resultSet.getObject(Columns.TYPE)).thenReturn("BATCH");
     when(resultSet.getString(Columns.DESCRIPTION)).thenReturn("DESCRIPTION");

diff --git a/api/src/test/resources/mappers/full_job_mapper.json b/api/src/test/resources/mappers/full_job_mapper.json
@@ -3,6 +3,7 @@
     "namespace": "NAMESPACE",
     "name": "NAME"
   },
+  "simpleName": "SIMPLE_NAME",
   "type": "BATCH",
   "description": "DESCRIPTION",
   "name": "NAME",

diff --git a/clients/java/src/main/java/marquez/client/models/Job.java b/clients/java/src/main/java/marquez/client/models/Job.java
@@ -22,6 +22,7 @@
 public final class Job extends JobMeta {
   @Getter private final JobId id;
   @Getter private final String name;
+  @Getter private final String simpleName;
   @Getter private final Instant createdAt;
   @Getter private final Instant updatedAt;
   @Getter private final String namespace;
@@ -33,6 +34,7 @@ public Job(
       @NonNull final JobId id,
       final JobType type,
       @NonNull final String name,
+      @NonNull final String simpleName,
       @NonNull final Instant createdAt,
       @NonNull final Instant updatedAt,
       @NonNull final String namespace,
@@ -47,6 +49,7 @@ public Job(
     super(type, inputs, outputs, location, context, description, null);
     this.id = id;
     this.name = name;
+    this.simpleName = simpleName;
     this.createdAt = createdAt;
     this.updatedAt = updatedAt;
     this.namespace = namespace;

diff --git a/clients/java/src/test/java/marquez/client/MarquezClientTest.java b/clients/java/src/test/java/marquez/client/MarquezClientTest.java
@@ -211,6 +211,7 @@ public class MarquezClientTest {
           JOB_ID,
           JOB_TYPE,
           JOB_NAME,
+          JOB_NAME,
           CREATED_AT,
           UPDATED_AT,
           NAMESPACE_NAME,
@@ -302,6 +303,7 @@ public class MarquezClientTest {
           JOB_ID,
           JOB_TYPE,
           JOB_NAME,
+          JOB_NAME,
           CREATED_AT,
           UPDATED_AT,
           NAMESPACE_NAME,

diff --git a/clients/java/src/test/java/marquez/client/models/JsonGenerator.java b/clients/java/src/test/java/marquez/client/models/JsonGenerator.java
@@ -270,6 +270,7 @@ public static String newJsonFor(final Job job) {
     obj.set("id", id);
     obj.put("type", job.getType().toString());
     obj.put("name", job.getName());
+    obj.put("simpleName", job.getName());
     obj.put("createdAt", ISO_INSTANT.format(job.getCreatedAt()));
     obj.put("updatedAt", ISO_INSTANT.format(job.getUpdatedAt()));
     obj.put("namespace", job.getNamespace());

diff --git a/clients/java/src/test/java/marquez/client/models/ModelGenerator.java b/clients/java/src/test/java/marquez/client/models/ModelGenerator.java
@@ -193,6 +193,7 @@ public static Job newJobWith(final Run latestRun, UUID currentVersion) {
         jobId,
         newJobType(),
         jobId.getName(),
+        jobId.getName(),
         now,
         now,
         jobId.getNamespace(),