From 32815a229bd2bb514f0b5fa83a4c656cd752f533 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Sun, 14 Jul 2024 17:23:39 +0530 Subject: [PATCH 1/8] missing dashboardInfo --- .../src/datahub/emitter/mcp_patch_builder.py | 10 ++ .../api/incremental_lineage_helper.py | 91 ++++++++++--- .../ingestion/source/powerbi/powerbi.py | 7 +- .../golden_test_admin_access_not_allowed.json | 43 +++++- .../integration/powerbi/golden_test_cll.json | 43 +++++- .../powerbi/golden_test_container.json | 125 ++++++++++++++++-- .../golden_test_disabled_ownership.json | 43 +++++- .../powerbi/golden_test_endorsement.json | 43 +++++- .../powerbi/golden_test_ingest.json | 43 +++++- .../powerbi/golden_test_lineage.json | 43 +++++- .../golden_test_lower_case_urn_ingest.json | 43 +++++- .../golden_test_platform_instance_ingest.json | 43 +++++- .../powerbi/golden_test_report.json | 95 ++++++++++++- .../golden_test_scan_all_workspaces.json | 73 ++++++++-- ...lden_test_server_to_platform_instance.json | 43 +++++- 15 files changed, 683 insertions(+), 105 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py index 3790399539437..da49688835a7a 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py @@ -71,6 +71,16 @@ def _add_patch( # TODO: Validate that aspectName is a valid aspect for this entityType self.patches[aspect_name].append(_Patch(op, path, value)) + def add_patch( + self, aspect_name: str, op: str, path: Union[str, Sequence[str]], value: Any + ) -> None: + return self._add_patch( + aspect_name=aspect_name, + op=op, + path=path, + value=value, + ) + def build(self) -> Iterable[MetadataChangeProposalClass]: return [ MetadataChangeProposalClass( diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 29e1f63dd452e..18dba594e4b12 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -1,4 +1,4 @@ -from typing import Iterable, Optional +from typing import ClassVar, Iterable, List, Optional, Union from pydantic.fields import Field @@ -11,6 +11,7 @@ DashboardInfoClass, FineGrainedLineageClass, MetadataChangeEventClass, + MetadataChangeProposalClass, SystemMetadataClass, UpstreamLineageClass, ) @@ -19,6 +20,72 @@ from datahub.specific.dataset import DatasetPatchBuilder +class PatchEntityAspect: + SKIPPABLE_ATTRIBUTES: ClassVar[List[str]] = [ + "ASPECT_INFO", + "ASPECT_NAME", + "ASPECT_TYPE", + "RECORD_SCHEMA", + ] + aspect: Union[ChartInfoClass, DashboardInfoClass] + patch_builder: DashboardPatchBuilder + attributes: List[str] + + def __init__( + self, + # The PatchEntityAspect can patch any Aspect, however to silent the lint Union is added for DashboardInfoClass + # We can use it with any Aspect + aspect: Union[DashboardInfoClass], + patch_builder: DashboardPatchBuilder, + ): + self.aspect = aspect + self.patch_builder = patch_builder + self.attributes = dir(self.aspect) + + def is_attribute_includable(self, attribute_name: str) -> bool: + """ + a child class can override this to add additional attributes to skip while generating patch aspect + """ + if ( + attribute_name.startswith("__") + or attribute_name.startswith("_") + or attribute_name in PatchEntityAspect.SKIPPABLE_ATTRIBUTES + ): + return False + + return True + + def attribute_path(self, attribute_name: str) -> str: + """ + a child class can override this if path is not equal to attribute_name + """ + return f"/{attribute_name}" + + def patch(self) -> Optional[MetadataChangeProposalClass]: + # filter property + properties = { + attr: getattr(self.aspect, attr) + for attr in self.attributes + if self.is_attribute_includable(attr) + and not callable(getattr(self.aspect, attr)) + } + + for property_ in properties: + if properties[property_]: + self.patch_builder.add_patch( + aspect_name=self.aspect.ASPECT_NAME, + op="add", + path=self.attribute_path(property_), + value=properties[property_], + ) + + mcps: List[MetadataChangeProposalClass] = list(self.patch_builder.build()) + if mcps: + return mcps[0] + + return None + + def convert_upstream_lineage_to_patch( urn: str, aspect: UpstreamLineageClass, @@ -62,26 +129,18 @@ def convert_dashboard_info_to_patch( ) -> Optional[MetadataWorkUnit]: patch_builder = DashboardPatchBuilder(urn, system_metadata) - if aspect.customProperties: - for key in aspect.customProperties: - patch_builder.add_custom_property( - key, str(aspect.customProperties.get(key)) - ) - - if aspect.datasetEdges: - for datasetEdge in aspect.datasetEdges: - patch_builder.add_dataset_edge(datasetEdge) + patch_entity_aspect: PatchEntityAspect = PatchEntityAspect( + aspect=aspect, + patch_builder=patch_builder, + ) - if aspect.chartEdges: - for chartEdge in aspect.chartEdges: - patch_builder.add_chart_edge(chartEdge) + mcp: Optional[MetadataChangeProposalClass] = patch_entity_aspect.patch() - values = patch_builder.build() - if values: - mcp = next(iter(values)) + if mcp: return MetadataWorkUnit( id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp ) + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index de4eaf6b64434..d761bc1e41014 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -1197,8 +1197,7 @@ def report_to_datahub_work_units( ) -> Iterable[MetadataWorkUnit]: mcps: List[MetadataChangeProposalWrapper] = [] - logger.debug(f"Converting dashboard={report.name} to datahub dashboard") - + logger.debug(f"Converting report={report.name} to datahub dashboard") # Convert user to CorpUser user_mcps = self.to_datahub_users(report.users) # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization @@ -1215,9 +1214,7 @@ def report_to_datahub_work_units( mcps.extend(chart_mcps) mcps.extend(report_mcps) - # Convert MCP to work_units - work_units = map(self._to_work_unit, mcps) - return work_units + return map(self._to_work_unit, mcps) @platform_name("PowerBI") diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json index 8ec431b6fe9f1..04a73ee422c44 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json @@ -300,18 +300,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 937cad0b9ec17..faaf386fc8d5a 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1264,18 +1264,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index 501ec284097b3..bd8ba5abfe828 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -1709,18 +1709,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, @@ -2903,6 +2934,58 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", @@ -3053,18 +3136,36 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "0" + "path": "/customProperties", + "value": { + "chartCount": "0", + "workspaceName": "second-demo-workspace", + "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" + } }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "second-demo-workspace" + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C22-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard2" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 74779ac7a0577..edaf06b42fe44 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -974,18 +974,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index 442dfd5c8c082..b4bb445dec2ba 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -1146,18 +1146,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 2c4ff6ee851f4..411cdee5a1e81 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1006,18 +1006,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 0aea8514559ec..a37110f0fdbdf 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -1189,18 +1189,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index 22a00236af8c6..db249682ab2a4 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -1006,18 +1006,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index cf5d4df460e23..4f87b839de7d9 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -1014,18 +1014,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index cce9e3f8755ba..088322195e398 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -1006,18 +1006,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, @@ -1935,6 +1966,58 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 5e244e0e0f93f..d273699829ab7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -974,18 +974,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, @@ -1075,18 +1106,36 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "0" + "path": "/customProperties", + "value": { + "chartCount": "0", + "workspaceName": "second-demo-workspace", + "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "second-demo-workspace" + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C22-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard2" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 8fd8989b81122..ba389ee1c7b3b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -1214,18 +1214,49 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/title", + "value": "test_dashboard" } ] }, From b73525934ac6595cdf3adbac16700d2fb38c5773 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Sun, 14 Jul 2024 17:44:14 +0530 Subject: [PATCH 2/8] golden file update --- .../powerbi/golden_test_admin_only.json | 82 +++++++++++++++++-- ..._config_and_modified_since_admin_only.json | 82 +++++++++++++++++-- 2 files changed, 152 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index 29e9ccebf067e..e9b569b356cc6 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -1214,18 +1214,44 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" } ] }, @@ -1951,6 +1977,50 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 11a7fed6030ce..2f2ed43ed6bed 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -1016,18 +1016,44 @@ "json": [ { "op": "add", - "path": "/customProperties/chartCount", - "value": "2" + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] }, { "op": "add", - "path": "/customProperties/workspaceName", - "value": "demo-workspace" + "path": "/customProperties", + "value": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + } + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" }, { "op": "add", - "path": "/customProperties/workspaceId", - "value": "64ED5CAD-7C10-4684-8180-826122881108" + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" } ] }, @@ -1257,6 +1283,50 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", From b3cb842fb606dbed2ee440ff4ca4b571c5204da8 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Sun, 14 Jul 2024 19:10:45 +0530 Subject: [PATCH 3/8] update chartInfo aspect --- .../api/incremental_lineage_helper.py | 49 +++++++++---------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 18dba594e4b12..0b1ad598756a9 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -5,6 +5,7 @@ from datahub.configuration.common import ConfigModel from datahub.emitter.mce_builder import datahub_guid, set_aspect from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.schema_classes import ( ChartInfoClass, @@ -28,15 +29,15 @@ class PatchEntityAspect: "RECORD_SCHEMA", ] aspect: Union[ChartInfoClass, DashboardInfoClass] - patch_builder: DashboardPatchBuilder + patch_builder: MetadataPatchProposal attributes: List[str] def __init__( self, # The PatchEntityAspect can patch any Aspect, however to silent the lint Union is added for DashboardInfoClass # We can use it with any Aspect - aspect: Union[DashboardInfoClass], - patch_builder: DashboardPatchBuilder, + aspect: Union[DashboardInfoClass, ChartInfoClass], + patch_builder: MetadataPatchProposal, ): self.aspect = aspect self.patch_builder = patch_builder @@ -100,30 +101,33 @@ def convert_upstream_lineage_to_patch( return MetadataWorkUnit(id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp) -def convert_chart_info_to_patch( - urn: str, aspect: ChartInfoClass, system_metadata: Optional[SystemMetadataClass] +def create_mw_for_patch_aspect( + patch_entity_aspect: PatchEntityAspect, ) -> Optional[MetadataWorkUnit]: - patch_builder = ChartPatchBuilder(urn, system_metadata) - - if aspect.customProperties: - for key in aspect.customProperties: - patch_builder.add_custom_property( - key, str(aspect.customProperties.get(key)) - ) - if aspect.inputEdges: - for inputEdge in aspect.inputEdges: - patch_builder.add_input_edge(inputEdge) + mcp: Optional[MetadataChangeProposalClass] = patch_entity_aspect.patch() - values = patch_builder.build() - if values: - mcp = next(iter(values)) + if mcp: return MetadataWorkUnit( id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp ) + return None +def convert_chart_info_to_patch( + urn: str, aspect: ChartInfoClass, system_metadata: Optional[SystemMetadataClass] +) -> Optional[MetadataWorkUnit]: + patch_builder = ChartPatchBuilder(urn, system_metadata) + + patch_entity_aspect: PatchEntityAspect = PatchEntityAspect( + aspect=aspect, + patch_builder=patch_builder, + ) + + return create_mw_for_patch_aspect(patch_entity_aspect=patch_entity_aspect) + + def convert_dashboard_info_to_patch( urn: str, aspect: DashboardInfoClass, system_metadata: Optional[SystemMetadataClass] ) -> Optional[MetadataWorkUnit]: @@ -134,14 +138,7 @@ def convert_dashboard_info_to_patch( patch_builder=patch_builder, ) - mcp: Optional[MetadataChangeProposalClass] = patch_entity_aspect.patch() - - if mcp: - return MetadataWorkUnit( - id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp - ) - - return None + return create_mw_for_patch_aspect(patch_entity_aspect=patch_entity_aspect) def get_fine_grained_lineage_key(fine_upstream: FineGrainedLineageClass) -> str: From 6ebed8c050261ab03b77df4d2fee258e17e83be1 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Thu, 18 Jul 2024 23:23:50 +0530 Subject: [PATCH 4/8] address review comments --- .../api/incremental_lineage_helper.py | 151 +++++++----------- .../src/datahub/specific/dashboard.py | 118 ++++++++++++++ .../golden_test_admin_access_not_allowed.json | 42 ++--- .../powerbi/golden_test_admin_only.json | 34 ++-- .../integration/powerbi/golden_test_cll.json | 42 ++--- .../powerbi/golden_test_container.json | 70 ++++---- .../golden_test_disabled_ownership.json | 42 ++--- .../powerbi/golden_test_endorsement.json | 42 ++--- .../powerbi/golden_test_ingest.json | 42 ++--- .../powerbi/golden_test_lineage.json | 42 ++--- .../golden_test_lower_case_urn_ingest.json | 42 ++--- ..._config_and_modified_since_admin_only.json | 34 ++-- .../golden_test_platform_instance_ingest.json | 42 ++--- .../powerbi/golden_test_report.json | 42 ++--- .../golden_test_scan_all_workspaces.json | 70 ++++---- ...lden_test_server_to_platform_instance.json | 42 ++--- 16 files changed, 540 insertions(+), 357 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 0b1ad598756a9..804b945a09a39 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -1,18 +1,16 @@ -from typing import ClassVar, Iterable, List, Optional, Union +from typing import Iterable, Optional from pydantic.fields import Field from datahub.configuration.common import ConfigModel from datahub.emitter.mce_builder import datahub_guid, set_aspect from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.schema_classes import ( ChartInfoClass, DashboardInfoClass, FineGrainedLineageClass, MetadataChangeEventClass, - MetadataChangeProposalClass, SystemMetadataClass, UpstreamLineageClass, ) @@ -21,72 +19,6 @@ from datahub.specific.dataset import DatasetPatchBuilder -class PatchEntityAspect: - SKIPPABLE_ATTRIBUTES: ClassVar[List[str]] = [ - "ASPECT_INFO", - "ASPECT_NAME", - "ASPECT_TYPE", - "RECORD_SCHEMA", - ] - aspect: Union[ChartInfoClass, DashboardInfoClass] - patch_builder: MetadataPatchProposal - attributes: List[str] - - def __init__( - self, - # The PatchEntityAspect can patch any Aspect, however to silent the lint Union is added for DashboardInfoClass - # We can use it with any Aspect - aspect: Union[DashboardInfoClass, ChartInfoClass], - patch_builder: MetadataPatchProposal, - ): - self.aspect = aspect - self.patch_builder = patch_builder - self.attributes = dir(self.aspect) - - def is_attribute_includable(self, attribute_name: str) -> bool: - """ - a child class can override this to add additional attributes to skip while generating patch aspect - """ - if ( - attribute_name.startswith("__") - or attribute_name.startswith("_") - or attribute_name in PatchEntityAspect.SKIPPABLE_ATTRIBUTES - ): - return False - - return True - - def attribute_path(self, attribute_name: str) -> str: - """ - a child class can override this if path is not equal to attribute_name - """ - return f"/{attribute_name}" - - def patch(self) -> Optional[MetadataChangeProposalClass]: - # filter property - properties = { - attr: getattr(self.aspect, attr) - for attr in self.attributes - if self.is_attribute_includable(attr) - and not callable(getattr(self.aspect, attr)) - } - - for property_ in properties: - if properties[property_]: - self.patch_builder.add_patch( - aspect_name=self.aspect.ASPECT_NAME, - op="add", - path=self.attribute_path(property_), - value=properties[property_], - ) - - mcps: List[MetadataChangeProposalClass] = list(self.patch_builder.build()) - if mcps: - return mcps[0] - - return None - - def convert_upstream_lineage_to_patch( urn: str, aspect: UpstreamLineageClass, @@ -101,44 +33,81 @@ def convert_upstream_lineage_to_patch( return MetadataWorkUnit(id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp) -def create_mw_for_patch_aspect( - patch_entity_aspect: PatchEntityAspect, +def convert_chart_info_to_patch( + urn: str, aspect: ChartInfoClass, system_metadata: Optional[SystemMetadataClass] ) -> Optional[MetadataWorkUnit]: + patch_builder = ChartPatchBuilder(urn, system_metadata) + + if aspect.customProperties: + for key in aspect.customProperties: + patch_builder.add_custom_property( + key, str(aspect.customProperties.get(key)) + ) - mcp: Optional[MetadataChangeProposalClass] = patch_entity_aspect.patch() + if aspect.inputEdges: + for inputEdge in aspect.inputEdges: + patch_builder.add_input_edge(inputEdge) - if mcp: + values = patch_builder.build() + if values: + mcp = next(iter(values)) return MetadataWorkUnit( id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp ) - return None -def convert_chart_info_to_patch( - urn: str, aspect: ChartInfoClass, system_metadata: Optional[SystemMetadataClass] +def convert_dashboard_info_to_patch( + urn: str, aspect: DashboardInfoClass, system_metadata: Optional[SystemMetadataClass] ) -> Optional[MetadataWorkUnit]: - patch_builder = ChartPatchBuilder(urn, system_metadata) + patch_builder = DashboardPatchBuilder(urn, system_metadata) - patch_entity_aspect: PatchEntityAspect = PatchEntityAspect( - aspect=aspect, - patch_builder=patch_builder, - ) + if aspect.customProperties: + for key in aspect.customProperties: + patch_builder.add_custom_property( + key, str(aspect.customProperties.get(key)) + ) - return create_mw_for_patch_aspect(patch_entity_aspect=patch_entity_aspect) + if aspect.datasetEdges: + for datasetEdge in aspect.datasetEdges: + patch_builder.add_dataset_edge(datasetEdge) + if aspect.chartEdges: + for chartEdge in aspect.chartEdges: + patch_builder.add_chart_edge(chartEdge) -def convert_dashboard_info_to_patch( - urn: str, aspect: DashboardInfoClass, system_metadata: Optional[SystemMetadataClass] -) -> Optional[MetadataWorkUnit]: - patch_builder = DashboardPatchBuilder(urn, system_metadata) + if aspect.title: + patch_builder.set_title(aspect.title) - patch_entity_aspect: PatchEntityAspect = PatchEntityAspect( - aspect=aspect, - patch_builder=patch_builder, - ) + if aspect.description: + patch_builder.set_description(aspect.description) + + if aspect.charts: + patch_builder.set_charts(aspect.charts) + + if aspect.dashboardUrl: + patch_builder.set_dashboard_url(aspect.dashboardUrl) - return create_mw_for_patch_aspect(patch_entity_aspect=patch_entity_aspect) + if aspect.datasets: + patch_builder.set_datasets(aspect.datasets) + + if aspect.access: + patch_builder.set_access(aspect.access) + + if aspect.lastRefreshed: + patch_builder.set_last_refreshed(aspect.lastRefreshed) + + if aspect.lastModified: + patch_builder.set_last_modified(last_modified=aspect.lastModified) + + values = patch_builder.build() + + if values: + mcp = next(iter(values)) + return MetadataWorkUnit( + id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp + ) + return None def get_fine_grained_lineage_key(fine_upstream: FineGrainedLineageClass) -> str: diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index e6d911b598655..437bdffad1b11 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -3,7 +3,9 @@ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( + AccessLevelClass, AuditStampClass, + ChangeAuditStampsClass, DashboardInfoClass as DashboardInfo, EdgeClass as Edge, GlobalTagsClass as GlobalTags, @@ -405,3 +407,119 @@ def remove_custom_property(self, key: str) -> "DashboardPatchBuilder": """ self.custom_properties_patch_helper.remove_property(key) return self + + def set_title(self, title: str) -> "DashboardPatchBuilder": + assert title, "DashboardInfo title should not be None" + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/title", + value=title, + ) + + return self + + def set_description(self, description: str) -> "DashboardPatchBuilder": + assert description, "DashboardInfo description should not be None" + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + + return self + + def add_custom_properties( + self, custom_properties: Optional[Dict[str, str]] = None + ) -> "DashboardPatchBuilder": + + if custom_properties: + for key, value in custom_properties.items(): + self.custom_properties_patch_helper.add_property(key, value) + + return self + + def set_external_url(self, external_url: Optional[str]) -> "DashboardPatchBuilder": + if external_url: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/externalUrl", + value=external_url, + ) + return self + + def set_charts(self, charts: Optional[List[str]]) -> "DashboardPatchBuilder": + if charts: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/charts", + value=charts, + ) + + return self + + def set_datasets(self, datasets: Optional[List[str]]) -> "DashboardPatchBuilder": + if datasets: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/datasets", + value=datasets, + ) + + return self + + def set_dashboard_url( + self, dashboard_url: Optional[str] + ) -> "DashboardPatchBuilder": + if dashboard_url: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/dashboardUrl", + value=dashboard_url, + ) + + return self + + def set_access( + self, access: Union[None, Union[str, "AccessLevelClass"]] = None + ) -> "DashboardPatchBuilder": + if access: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/access", + value=access, + ) + + return self + + def set_last_refreshed( + self, last_refreshed: Optional[int] + ) -> "DashboardPatchBuilder": + if last_refreshed: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/lastRefreshed", + value=last_refreshed, + ) + + return self + + def set_last_modified( + self, last_modified: "ChangeAuditStampsClass" + ) -> "DashboardPatchBuilder": + if last_modified: + self._add_patch( + DashboardInfo.ASPECT_NAME, + "add", + path="/lastModified", + value=last_modified, + ) + + return self diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json index 04a73ee422c44..60771ccf4aaf1 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json @@ -300,31 +300,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -338,11 +349,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index e9b569b356cc6..c3dd27d4ca18d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -1212,6 +1212,26 @@ "aspectName": "dashboardInfo", "aspect": { "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, { "op": "add", "path": "/charts", @@ -1220,15 +1240,6 @@ "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" ] }, - { - "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } - }, { "op": "add", "path": "/dashboardUrl", @@ -1247,11 +1258,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index faaf386fc8d5a..4138c964806ae 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1264,31 +1264,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1302,11 +1313,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index bd8ba5abfe828..397962a721bc7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -1709,31 +1709,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1747,11 +1758,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, @@ -3136,12 +3142,23 @@ "json": [ { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "0", - "workspaceName": "second-demo-workspace", - "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" - } + "path": "/customProperties/chartCount", + "value": "0" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "second-demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C22-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard2" }, { "op": "add", @@ -3161,11 +3178,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard2" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index edaf06b42fe44..5b2f392f3ac9a 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -974,31 +974,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1012,11 +1023,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index b4bb445dec2ba..7b6a35b354d62 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -1146,31 +1146,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1184,11 +1195,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 411cdee5a1e81..5517ffb46e173 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1006,31 +1006,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1044,11 +1055,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index a37110f0fdbdf..7a7b5bcb37c79 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -1189,31 +1189,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1227,11 +1238,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index db249682ab2a4..f5bf418e58346 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -1006,31 +1006,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1044,11 +1055,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 2f2ed43ed6bed..6d14b27d3fce9 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -1014,6 +1014,26 @@ "aspectName": "dashboardInfo", "aspect": { "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "2" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, { "op": "add", "path": "/charts", @@ -1022,15 +1042,6 @@ "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" ] }, - { - "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } - }, { "op": "add", "path": "/dashboardUrl", @@ -1049,11 +1060,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 4f87b839de7d9..14ba7bd6c7b9d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -1014,31 +1014,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1052,11 +1063,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 088322195e398..17133951749d9 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -1006,31 +1006,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1044,11 +1055,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index d273699829ab7..580f9a766110b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -974,31 +974,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1012,11 +1023,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, @@ -1106,12 +1112,23 @@ "json": [ { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "0", - "workspaceName": "second-demo-workspace", - "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" - } + "path": "/customProperties/chartCount", + "value": "0" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "second-demo-workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C22-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard2" }, { "op": "add", @@ -1131,11 +1148,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard2" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index ba389ee1c7b3b..0940bd2917baa 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -1214,31 +1214,42 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/customProperties/chartCount", + "value": "2" }, { "op": "add", - "path": "/customProperties", - "value": { - "chartCount": "2", - "workspaceName": "demo-workspace", - "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" - } + "path": "/customProperties/workspaceName", + "value": "demo-workspace" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://localhost/dashboards/web/1" + "path": "/customProperties/workspaceId", + "value": "64ED5CAD-7C10-4684-8180-826122881108" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" }, { "op": "add", "path": "/description", "value": "Description of test dashboard" }, + { + "op": "add", + "path": "/charts", + "value": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ] + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, { "op": "add", "path": "/lastModified", @@ -1252,11 +1263,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "test_dashboard" } ] }, From 0e65348203c9fcc965f43ecf580b024fb7d1f4eb Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Thu, 18 Jul 2024 23:41:34 +0530 Subject: [PATCH 5/8] remove unused method --- .../src/datahub/emitter/mcp_patch_builder.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py index da49688835a7a..3790399539437 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py @@ -71,16 +71,6 @@ def _add_patch( # TODO: Validate that aspectName is a valid aspect for this entityType self.patches[aspect_name].append(_Patch(op, path, value)) - def add_patch( - self, aspect_name: str, op: str, path: Union[str, Sequence[str]], value: Any - ) -> None: - return self._add_patch( - aspect_name=aspect_name, - op=op, - path=path, - value=value, - ) - def build(self) -> Iterable[MetadataChangeProposalClass]: return [ MetadataChangeProposalClass( From ddf124a390dd1a107d1743d18269a6da708a2fbb Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Sun, 21 Jul 2024 18:55:27 +0530 Subject: [PATCH 6/8] log message --- .../src/datahub/ingestion/api/incremental_lineage_helper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 804b945a09a39..f2049b9b382ee 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -1,3 +1,4 @@ +import logging from typing import Iterable, Optional from pydantic.fields import Field @@ -18,6 +19,8 @@ from datahub.specific.dashboard import DashboardPatchBuilder from datahub.specific.dataset import DatasetPatchBuilder +logger = logging.getLogger(__name__) + def convert_upstream_lineage_to_patch( urn: str, @@ -103,6 +106,9 @@ def convert_dashboard_info_to_patch( values = patch_builder.build() if values: + logger.debug( + f"Generating patch DashboardInfo MetadataWorkUnit for dashboard {aspect.title}" + ) mcp = next(iter(values)) return MetadataWorkUnit( id=MetadataWorkUnit.generate_workunit_id(mcp), mcp_raw=mcp From f5672583f0849c68450b6b69797851d5b9f002b5 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Wed, 24 Jul 2024 12:58:52 +0530 Subject: [PATCH 7/8] improve patch metadata --- .../api/incremental_lineage_helper.py | 4 +- .../ingestion/source/powerbi/config.py | 5 + .../ingestion/source/powerbi/powerbi.py | 2 +- .../src/datahub/specific/dashboard.py | 34 +- .../golden_test_admin_access_not_allowed.json | 12 +- .../powerbi/golden_test_admin_only.json | 12 +- .../integration/powerbi/golden_test_cll.json | 12 +- .../powerbi/golden_test_container.json | 42 +- .../golden_test_disabled_ownership.json | 12 +- .../powerbi/golden_test_endorsement.json | 12 +- .../powerbi/golden_test_ingest.json | 12 +- .../golden_test_ingest_patch_disabled.json | 1153 +++++++++++++++++ .../powerbi/golden_test_lineage.json | 12 +- .../golden_test_lower_case_urn_ingest.json | 12 +- ..._config_and_modified_since_admin_only.json | 12 +- .../golden_test_platform_instance_ingest.json | 12 +- .../powerbi/golden_test_report.json | 42 +- .../golden_test_scan_all_workspaces.json | 12 +- ...lden_test_server_to_platform_instance.json | 12 +- .../tests/integration/powerbi/test_powerbi.py | 45 + 20 files changed, 1354 insertions(+), 117 deletions(-) create mode 100644 metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index f2049b9b382ee..66041f9edaa76 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -86,13 +86,13 @@ def convert_dashboard_info_to_patch( patch_builder.set_description(aspect.description) if aspect.charts: - patch_builder.set_charts(aspect.charts) + patch_builder.add_charts(aspect.charts) if aspect.dashboardUrl: patch_builder.set_dashboard_url(aspect.dashboardUrl) if aspect.datasets: - patch_builder.set_datasets(aspect.datasets) + patch_builder.add_datasets(aspect.datasets) if aspect.access: patch_builder.set_access(aspect.access) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index bd80433bc2e6c..967dd5d81112d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -440,6 +440,11 @@ class PowerBiDashboardSourceConfig( ) profiling: PowerBiProfilingConfig = PowerBiProfilingConfig() + patch_metadata: bool = pydantic.Field( + default=True, + description="Patch dashboard metadata", + ) + @root_validator(skip_on_failure=True) def validate_extract_column_level_lineage(cls, values: Dict) -> Dict: flags = [ diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index d761bc1e41014..73f242a06b1d6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -1382,7 +1382,7 @@ def _get_dashboard_patch_work_unit( DashboardInfoClass ] = work_unit.get_aspect_of_type(DashboardInfoClass) - if dashboard_info_aspect: + if dashboard_info_aspect and self.source_config.patch_metadata: return convert_dashboard_info_to_patch( work_unit.get_urn(), dashboard_info_aspect, diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index 437bdffad1b11..4018b28770b13 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -450,25 +450,27 @@ def set_external_url(self, external_url: Optional[str]) -> "DashboardPatchBuilde ) return self - def set_charts(self, charts: Optional[List[str]]) -> "DashboardPatchBuilder": - if charts: - self._add_patch( - DashboardInfo.ASPECT_NAME, - "add", - path="/charts", - value=charts, - ) + def add_charts(self, chart_urns: Optional[List[str]]) -> "DashboardPatchBuilder": + if chart_urns: + for urn in chart_urns: + self._add_patch( + aspect_name=DashboardInfo.ASPECT_NAME, + op="add", + path=f"/charts/{urn}", + value=urn + ) return self - def set_datasets(self, datasets: Optional[List[str]]) -> "DashboardPatchBuilder": - if datasets: - self._add_patch( - DashboardInfo.ASPECT_NAME, - "add", - path="/datasets", - value=datasets, - ) + def add_datasets(self, dataset_urns: Optional[List[str]]) -> "DashboardPatchBuilder": + if dataset_urns: + for urn in dataset_urns: + self._add_patch( + aspect_name=DashboardInfo.ASPECT_NAME, + op="add", + path=f"/datasets/{urn}", + value=urn, + ) return self diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json index 60771ccf4aaf1..049008bddc58e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json @@ -325,11 +325,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index c3dd27d4ca18d..fa4bcb8abaa94 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -1234,11 +1234,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 4138c964806ae..60b36897ed2e4 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1289,11 +1289,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index 397962a721bc7..b43e4a6c2c1c2 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -1734,11 +1734,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", @@ -2949,21 +2951,28 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" - ] + "path": "/title", + "value": "SalesMarketing" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + "path": "/description", + "value": "Acryl sales marketing report" }, { "op": "add", - "path": "/description", - "value": "Acryl sales marketing report" + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" }, { "op": "add", @@ -2978,11 +2987,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "SalesMarketing" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 5b2f392f3ac9a..c5414444cc35b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -999,11 +999,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index 7b6a35b354d62..e1ddbfb901bad 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -1171,11 +1171,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 5517ffb46e173..6f899a7fa11b7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1031,11 +1031,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json new file mode 100644 index 0000000000000..efbd9abfdb911 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json @@ -0,0 +1,1153 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "public issue_history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "SNOWFLAKE_TESTTABLE", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "big-query-with-parameter", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"AUPRDWHDB\", \"COMMOPSDB\", [Query=\"select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,'-',''))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION\", CommandTimeout=#duration(0, 1, 30, 0)]),\n #\"Changed Type\" = Table.TransformColumnTypes(Source,{{\"mth_date\", type date}}),\n #\"Added Custom\" = Table.AddColumn(#\"Changed Type\", \"Month\", each Date.Month([mth_date])),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"TPV Opening\", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #\"Added Custom1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "ms_sql_native_table", + "description": "hr pbi test description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Dataset Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User1@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User2@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details" + }, + "title": "test_tile", + "description": "test_tile", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "createdFrom": "Dataset", + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed", + "datasetWebUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details" + }, + "title": "yearly_sales", + "description": "yearly_sales", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Tile" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "chartCount": "2", + "workspaceName": "demo-workspace", + "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" + }, + "title": "test_dashboard", + "description": "Description of test dashboard", + "charts": [ + "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "dashboardUrl": "https://localhost/dashboards/web/1" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 7a7b5bcb37c79..9a09cb4fec64d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -1214,11 +1214,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index f5bf418e58346..d80aa02c4cb12 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -1031,11 +1031,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 6d14b27d3fce9..66e87952bf141 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -1036,11 +1036,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index 14ba7bd6c7b9d..ea1ee0df4b105 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -1039,11 +1039,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,aws-ap-south-1.charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 17133951749d9..094869bfd24f1 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -1031,11 +1031,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", @@ -1981,21 +1983,28 @@ "json": [ { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", - "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" - ] + "path": "/title", + "value": "SalesMarketing" }, { "op": "add", - "path": "/dashboardUrl", - "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + "path": "/description", + "value": "Acryl sales marketing report" }, { "op": "add", - "path": "/description", - "value": "Acryl sales marketing report" + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" }, { "op": "add", @@ -2010,11 +2019,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "SalesMarketing" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index 580f9a766110b..dcaa518a3c323 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -999,11 +999,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index 0940bd2917baa..bc5e844f679c7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -1239,11 +1239,13 @@ }, { "op": "add", - "path": "/charts", - "value": [ - "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" - ] + "path": "/charts/urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "value": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", + "value": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" }, { "op": "add", diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 06cc40fe7b24c..064cee2507d51 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -683,6 +683,51 @@ def test_powerbi_ingest( ) +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +@pytest.mark.integration +def test_powerbi_ingest_patch_disabled( + mock_msal: MagicMock, + pytestconfig: pytest.Config, + tmp_path: str, + mock_time: datetime.datetime, + requests_mock: Any, +) -> None: + enable_logging() + + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api(request_mock=requests_mock) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + "patch_metadata": False, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_ingest_patch_disabled.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/powerbi_mces.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) + @freeze_time(FROZEN_TIME) @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) @pytest.mark.integration From 4130d5b02fc149a5ed57c9bfdaf7830e0926aec5 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Wed, 24 Jul 2024 22:53:33 +0530 Subject: [PATCH 8/8] address review comments --- .../api/incremental_lineage_helper.py | 14 +++ .../src/datahub/specific/chart.py | 108 ++++++++++++++++++ .../src/datahub/specific/dashboard.py | 6 +- .../tests/integration/powerbi/test_powerbi.py | 1 + .../test_incremental_lineage_helper.py | 40 +++++++ 5 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py diff --git a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py index 66041f9edaa76..78a091f1ffe68 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py +++ b/metadata-ingestion/src/datahub/ingestion/api/incremental_lineage_helper.py @@ -51,6 +51,20 @@ def convert_chart_info_to_patch( for inputEdge in aspect.inputEdges: patch_builder.add_input_edge(inputEdge) + patch_builder.set_chart_url(aspect.chartUrl).set_external_url( + aspect.externalUrl + ).set_type(aspect.type).set_title(aspect.title).set_access( + aspect.access + ).set_last_modified( + aspect.lastModified + ).set_last_refreshed( + aspect.lastRefreshed + ).set_description( + aspect.description + ).add_inputs( + aspect.inputs + ) + values = patch_builder.build() if values: mcp = next(iter(values)) diff --git a/metadata-ingestion/src/datahub/specific/chart.py b/metadata-ingestion/src/datahub/specific/chart.py index 51994ad1d063e..cc68168b68db7 100644 --- a/metadata-ingestion/src/datahub/specific/chart.py +++ b/metadata-ingestion/src/datahub/specific/chart.py @@ -3,8 +3,11 @@ from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( + AccessLevelClass, AuditStampClass, + ChangeAuditStampsClass, ChartInfoClass as ChartInfo, + ChartTypeClass, EdgeClass as Edge, GlobalTagsClass as GlobalTags, GlossaryTermAssociationClass as Term, @@ -311,3 +314,108 @@ def remove_custom_property(self, key: str) -> "ChartPatchBuilder": """ self.custom_properties_patch_helper.remove_property(key) return self + + def set_title(self, title: str) -> "ChartPatchBuilder": + assert title, "ChartInfo title should not be None" + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/title", + value=title, + ) + + return self + + def set_description(self, description: str) -> "ChartPatchBuilder": + assert description, "DashboardInfo description should not be None" + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/description", + value=description, + ) + + return self + + def set_last_refreshed(self, last_refreshed: Optional[int]) -> "ChartPatchBuilder": + if last_refreshed: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/lastRefreshed", + value=last_refreshed, + ) + + return self + + def set_last_modified( + self, last_modified: "ChangeAuditStampsClass" + ) -> "ChartPatchBuilder": + if last_modified: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/lastModified", + value=last_modified, + ) + + return self + + def set_external_url(self, external_url: Optional[str]) -> "ChartPatchBuilder": + if external_url: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/externalUrl", + value=external_url, + ) + return self + + def set_chart_url(self, dashboard_url: Optional[str]) -> "ChartPatchBuilder": + if dashboard_url: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/chartUrl", + value=dashboard_url, + ) + + return self + + def set_type( + self, type: Union[None, Union[str, "ChartTypeClass"]] = None + ) -> "ChartPatchBuilder": + if type: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/type", + value=type, + ) + + return self + + def set_access( + self, access: Union[None, Union[str, "AccessLevelClass"]] = None + ) -> "ChartPatchBuilder": + if access: + self._add_patch( + ChartInfo.ASPECT_NAME, + "add", + path="/access", + value=access, + ) + + return self + + def add_inputs(self, input_urns: Optional[List[str]]) -> "ChartPatchBuilder": + if input_urns: + for urn in input_urns: + self._add_patch( + aspect_name=ChartInfo.ASPECT_NAME, + op="add", + path=f"/inputs/{urn}", + value=urn, + ) + + return self diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index 4018b28770b13..8228dbc011db2 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -457,12 +457,14 @@ def add_charts(self, chart_urns: Optional[List[str]]) -> "DashboardPatchBuilder" aspect_name=DashboardInfo.ASPECT_NAME, op="add", path=f"/charts/{urn}", - value=urn + value=urn, ) return self - def add_datasets(self, dataset_urns: Optional[List[str]]) -> "DashboardPatchBuilder": + def add_datasets( + self, dataset_urns: Optional[List[str]] + ) -> "DashboardPatchBuilder": if dataset_urns: for urn in dataset_urns: self._add_patch( diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 064cee2507d51..6a95ec2c1dda4 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -728,6 +728,7 @@ def test_powerbi_ingest_patch_disabled( golden_path=f"{test_resources_dir}/{golden_file}", ) + @freeze_time(FROZEN_TIME) @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) @pytest.mark.integration diff --git a/metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py new file mode 100644 index 0000000000000..1db4e48fce608 --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_incremental_lineage_helper.py @@ -0,0 +1,40 @@ +from typing import Optional + +from datahub.ingestion.api.incremental_lineage_helper import convert_chart_info_to_patch +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.metadata.schema_classes import ( + ChangeAuditStampsClass, + ChartInfoClass, + MetadataChangeProposalClass, +) + + +def test_convert_chart_info_to_patch(): + chart_info_class: ChartInfoClass = ChartInfoClass( + title="foo", + description="Checking patch", + inputs=[ + "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + ], + lastModified=ChangeAuditStampsClass(), + ) + + mw: Optional[MetadataWorkUnit] = convert_chart_info_to_patch( + urn="urn:li:chart:(looker,dashboard_elements.1)", + aspect=chart_info_class, + system_metadata=None, + ) + + assert mw + + assert mw.id == "urn:li:chart:(looker,dashboard_elements.1)-chartInfo" + + assert isinstance(mw.metadata, MetadataChangeProposalClass) + + assert mw.metadata.aspect + + assert ( + mw.metadata.aspect.value + == b'[{"op": "add", "path": "/title", "value": "foo"}, {"op": "add", "path": "/lastModified", "value": {"created": {"time": 0, "actor": "urn:li:corpuser:unknown"}, "lastModified": {"time": 0, "actor": "urn:li:corpuser:unknown"}}}, {"op": "add", "path": "/description", "value": "Checking patch"}, {"op": "add", "path": "/inputs/urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)", "value": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)"}, {"op": "add", "path": "/inputs/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", "value": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)"}]' + )