fix(timeline api): adding modification category (datahub-project#11345)

Co-authored-by: Chris Collins <chriscollins3456@gmail.com>
sleeperdeep · Dec 17, 2024 · 1c0bf35 · 1c0bf35
1 parent 7311802
commit 1c0bf35
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 29 deletions.
diff --git a/.../main/java/com/linkedin/metadata/timeline/data/dataset/DatasetSchemaFieldChangeEvent.java b/.../main/java/com/linkedin/metadata/timeline/data/dataset/DatasetSchemaFieldChangeEvent.java
@@ -27,16 +27,24 @@ public DatasetSchemaFieldChangeEvent(
       String description,
       String fieldPath,
       Urn fieldUrn,
-      boolean nullable) {
+      boolean nullable,
+      SchemaFieldModificationCategory modificationCategory) {
     super(
         entityUrn,
         category,
         operation,
         modifier,
         ImmutableMap.of(
-            "fieldPath", fieldPath,
-            "fieldUrn", fieldUrn.toString(),
-            "nullable", nullable),
+            "fieldPath",
+            fieldPath,
+            "fieldUrn",
+            fieldUrn.toString(),
+            "nullable",
+            nullable,
+            "modificationCategory",
+            modificationCategory != null
+                ? modificationCategory.toString()
+                : SchemaFieldModificationCategory.OTHER.toString()),
         auditStamp,
         semVerChange,
         description);

diff --git a/...ain/java/com/linkedin/metadata/timeline/data/dataset/SchemaFieldModificationCategory.java b/...ain/java/com/linkedin/metadata/timeline/data/dataset/SchemaFieldModificationCategory.java
@@ -0,0 +1,13 @@
+package com.linkedin.metadata.timeline.data.dataset;
+
+/*
+ * Enum to allow us to distinguish between the different schema field modifications when creating entity change events.
+ */
+public enum SchemaFieldModificationCategory {
+  // when a schema field is renamed
+  RENAME,
+  // when a schema field has a type change
+  TYPE_CHANGE,
+  // a default option when no other modification category has been given
+  OTHER,
+}
diff --git a/...ava/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java b/...ava/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java
@@ -15,6 +15,7 @@
 import com.linkedin.metadata.timeline.data.ChangeTransaction;
 import com.linkedin.metadata.timeline.data.SemanticChangeType;
 import com.linkedin.metadata.timeline.data.dataset.DatasetSchemaFieldChangeEvent;
+import com.linkedin.metadata.timeline.data.dataset.SchemaFieldModificationCategory;
 import com.linkedin.schema.SchemaField;
 import com.linkedin.schema.SchemaFieldArray;
 import com.linkedin.schema.SchemaMetadata;
@@ -246,6 +247,7 @@ private static void processFieldPathDataTypeChange(
                 .fieldPath(curBaseField.getFieldPath())
                 .fieldUrn(getSchemaFieldUrn(datasetUrn, curBaseField))
                 .nullable(curBaseField.isNullable())
+                .modificationCategory(SchemaFieldModificationCategory.TYPE_CHANGE)
                 .auditStamp(auditStamp)
                 .build());
       }
@@ -483,6 +485,7 @@ private static ChangeEvent generateRenameEvent(
         .fieldPath(curBaseField.getFieldPath())
         .fieldUrn(getSchemaFieldUrn(datasetUrn, curBaseField))
         .nullable(curBaseField.isNullable())
+        .modificationCategory(SchemaFieldModificationCategory.RENAME)
         .auditStamp(auditStamp)
         .build();
   }

diff --git a/...com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java b/...com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGeneratorTest.java
@@ -5,6 +5,7 @@
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.metadata.timeline.data.ChangeEvent;
+import com.linkedin.metadata.timeline.data.dataset.SchemaFieldModificationCategory;
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.restli.internal.server.util.DataMapUtils;
 import com.linkedin.schema.SchemaField;
@@ -42,6 +43,17 @@ private static void compareDescriptions(
     assertEquals(expectedDescriptions, actualDescriptions);
   }
 
+  private static void compareModificationCategories(
+      Set<String> expectedCategories, List<ChangeEvent> actual) {
+    Set<String> actualModificationCategories = new HashSet<>();
+    actual.forEach(
+        changeEvent -> {
+          actualModificationCategories.add(
+              changeEvent.getParameters().get("modificationCategory").toString());
+        });
+    assertEquals(expectedCategories, actualModificationCategories);
+  }
+
   private static Aspect<SchemaMetadata> getSchemaMetadata(List<SchemaField> schemaFieldList) {
     return new Aspect<>(
         new SchemaMetadata().setFields(new SchemaFieldArray(schemaFieldList)),
@@ -70,14 +82,17 @@ public void testNativeSchemaBackwardIncompatibleChange() throws Exception {
         Set.of(
             "A backwards incompatible change due to native datatype of the field 'ID' changed from 'NUMBER(16,1)' to 'NUMBER(10,1)'."),
         actual);
-
+    compareModificationCategories(
+        Set.of(SchemaFieldModificationCategory.TYPE_CHANGE.toString()), actual);
     List<ChangeEvent> actual2 = test.getChangeEvents(urn, entity, aspect, to, from, auditStamp);
     // Test single field going from NUMBER(10,1) -> NUMBER(16,1)
     assertEquals(1, actual2.size());
     compareDescriptions(
         Set.of(
             "A backwards incompatible change due to native datatype of the field 'ID' changed from 'NUMBER(10,1)' to 'NUMBER(16,1)'."),
         actual2);
+    compareModificationCategories(
+        Set.of(SchemaFieldModificationCategory.TYPE_CHANGE.toString()), actual);
   }
 
   @Test
@@ -104,6 +119,11 @@ public void testNativeSchemaFieldAddition() throws Exception {
             "A backwards incompatible change due to native datatype of the field 'ID' changed from 'NUMBER(16,1)' to 'NUMBER(10,1)'.",
             "A forwards & backwards compatible change due to the newly added field 'aa'."),
         actual);
+    compareModificationCategories(
+        Set.of(
+            SchemaFieldModificationCategory.TYPE_CHANGE.toString(),
+            SchemaFieldModificationCategory.OTHER.toString()),
+        actual);
   }
 
   @Test
@@ -127,6 +147,8 @@ public void testSchemaFieldRename() throws Exception {
             "A forwards & backwards compatible change due to renaming of the field 'ID to ID2'."),
         actual);
     assertEquals(1, actual.size());
+    compareModificationCategories(
+        Set.of(SchemaFieldModificationCategory.RENAME.toString()), actual);
   }
 
   @Test

diff --git a/smoke-test/tests/openapi/v1/timeline.json b/smoke-test/tests/openapi/v1/timeline.json
@@ -351,7 +351,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service'."
@@ -364,7 +365,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.type)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.type'."
@@ -377,7 +379,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider'."
@@ -390,7 +393,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.id)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.id'."
@@ -403,7 +407,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.name)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.name'."
@@ -416,7 +421,8 @@
               "parameters": {
                 "fieldPath": "property_id",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),property_id)",
-                "nullable": false
+                "nullable": false,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'property_id'."
@@ -438,7 +444,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.id)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "RENAME"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to renaming of the field 'service.provider.id to service.provider.id2'."
@@ -451,7 +458,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id3",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.id3)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.id3'."
@@ -464,7 +472,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.name)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MAJOR",
               "description": "A backwards incompatible change due to removal of field: 'service.provider.name'."
@@ -486,7 +495,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id2",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.id2)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "RENAME"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to renaming of the field 'service.provider.id2 to service.provider.id'."
@@ -499,7 +509,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id3",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.id3)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MAJOR",
               "description": "A backwards incompatible change due to removal of field: 'service.provider.id3'."
@@ -512,7 +523,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV1,PROD),service.provider.name)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.name'."

diff --git a/smoke-test/tests/openapi/v2/timeline.json b/smoke-test/tests/openapi/v2/timeline.json
@@ -351,7 +351,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service'."
@@ -364,7 +365,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.type)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.type'."
@@ -377,7 +379,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider'."
@@ -390,7 +393,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.id)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.id'."
@@ -403,7 +407,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.name)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.name'."
@@ -416,7 +421,8 @@
               "parameters": {
                 "fieldPath": "property_id",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),property_id)",
-                "nullable": false
+                "nullable": false,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'property_id'."
@@ -438,7 +444,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.id)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "RENAME"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to renaming of the field 'service.provider.id to service.provider.id2'."
@@ -451,7 +458,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id3",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.id3)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.id3'."
@@ -464,7 +472,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.name)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MAJOR",
               "description": "A backwards incompatible change due to removal of field: 'service.provider.name'."
@@ -486,7 +495,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id2",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.id2)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "RENAME"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to renaming of the field 'service.provider.id2 to service.provider.id'."
@@ -499,7 +509,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id3",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.id3)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MAJOR",
               "description": "A backwards incompatible change due to removal of field: 'service.provider.id3'."
@@ -512,7 +523,8 @@
               "parameters": {
                 "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name",
                 "fieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:test,datasetTimelineV2,PROD),service.provider.name)",
-                "nullable": true
+                "nullable": true,
+                "modificationCategory": "OTHER"
               },
               "semVerChange": "MINOR",
               "description": "A forwards & backwards compatible change due to the newly added field 'service.provider.name'."