Merge branch 'master' into ing-655-tableau-error-message

README.md

@@ -118,6 +118,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Cabify](https://cabify.tech/)
 - [ClassDojo](https://www.classdojo.com/)
 - [Coursera](https://www.coursera.org/)
+- [CVS Health](https://www.cvshealth.com/)
 - [DefinedCrowd](http://www.definedcrowd.com)
 - [DFDS](https://www.dfds.com/)
 - [Digital Turbine](https://www.digitalturbine.com/)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/entitytype/EntityTypeMapper.java

@@ -48,6 +48,7 @@ public class EntityTypeMapper {
           .put(EntityType.BUSINESS_ATTRIBUTE, Constants.BUSINESS_ATTRIBUTE_ENTITY_NAME)
           .put(EntityType.QUERY, Constants.QUERY_ENTITY_NAME)
           .put(EntityType.POST, Constants.POST_ENTITY_NAME)
+          .put(EntityType.FORM, Constants.FORM_ENTITY_NAME)
           .build();
 
   private static final Map<String, EntityType> ENTITY_NAME_TO_TYPE =

datahub-web-react/src/Mocks.tsx

@@ -1254,6 +1254,7 @@ export const glossaryNode5 = {
 
 export const sampleTag = {
     urn: 'urn:li:tag:abc-sample-tag',
+    type: EntityType.Tag,
     name: 'abc-sample-tag',
     description: 'sample tag description',
     ownership: {

datahub-web-react/src/app/ingest/source/utils.ts

@@ -47,6 +47,7 @@ export const WARNING = 'WARNING';
 export const FAILURE = 'FAILURE';
 export const CONNECTION_FAILURE = 'CONNECTION_FAILURE';
 export const CANCELLED = 'CANCELLED';
+export const ABORTED = 'ABORTED';
 export const UP_FOR_RETRY = 'UP_FOR_RETRY';
 export const ROLLING_BACK = 'ROLLING_BACK';
 export const ROLLED_BACK = 'ROLLED_BACK';
@@ -68,6 +69,7 @@ export const getExecutionRequestStatusIcon = (status: string) => {
         (status === ROLLED_BACK && WarningOutlined) ||
         (status === ROLLING_BACK && LoadingOutlined) ||
         (status === ROLLBACK_FAILED && CloseCircleOutlined) ||
+        (status === ABORTED && CloseCircleOutlined) ||
         ClockCircleOutlined
     );
 };
@@ -83,6 +85,7 @@ export const getExecutionRequestStatusDisplayText = (status: string) => {
         (status === ROLLED_BACK && 'Rolled Back') ||
         (status === ROLLING_BACK && 'Rolling Back') ||
         (status === ROLLBACK_FAILED && 'Rollback Failed') ||
+        (status === ABORTED && 'Aborted') ||
         status
     );
 };
@@ -105,6 +108,8 @@ export const getExecutionRequestSummaryText = (status: string) => {
             return 'Ingestion is in the process of rolling back.';
         case ROLLBACK_FAILED:
             return 'Ingestion rollback failed.';
+        case ABORTED:
+            return 'Ingestion job got aborted due to worker restart.';
         default:
             return 'Ingestion status not recognized.';
     }
@@ -121,6 +126,7 @@ export const getExecutionRequestStatusDisplayColor = (status: string) => {
         (status === ROLLED_BACK && 'orange') ||
         (status === ROLLING_BACK && 'orange') ||
         (status === ROLLBACK_FAILED && 'red') ||
+        (status === ABORTED && 'red') ||
         ANTD_GRAY[7]
     );
 };

datahub-web-react/src/graphql/container.graphql

@@ -1,6 +1,7 @@
 query getContainer($urn: String!) {
     container(urn: $urn) {
         urn
+        type
         exists
         lastIngested
         platform {

datahub-web-react/src/graphql/dashboard.graphql

@@ -1,5 +1,7 @@
 query getDashboard($urn: String!) {
     dashboard(urn: $urn) {
+        urn
+        type
         ...dashboardFields
         privileges {
             ...entityPrivileges

datahub-web-react/src/graphql/dataJob.graphql

@@ -1,5 +1,7 @@
 query getDataJob($urn: String!) {
     dataJob(urn: $urn) {
+        urn
+        type
         ...dataJobFields
         privileges {
             ...entityPrivileges

datahub-web-react/src/graphql/dataPlatform.graphql

@@ -1,5 +1,7 @@
 query getDataPlatform($urn: String!) {
     dataPlatform(urn: $urn) {
+        urn
+        type
         ...platformFields
     }
 }

datahub-web-react/src/graphql/dataProduct.graphql

@@ -1,5 +1,7 @@
 query getDataProduct($urn: String!) {
     dataProduct(urn: $urn) {
+        urn
+        type
         ...dataProductFields
         privileges {
             ...entityPrivileges

datahub-web-react/src/graphql/dataset.graphql

@@ -1,6 +1,7 @@
 query getDataProfiles($urn: String!, $limit: Int, $startTime: Long, $endTime: Long) {
     dataset(urn: $urn) {
         urn
+        type
         datasetProfiles(limit: $limit, startTimeMillis: $startTime, endTimeMillis: $endTime) {
             rowCount
             columnCount

datahub-web-react/src/graphql/mlFeature.graphql

@@ -1,5 +1,7 @@
 query getMLFeature($urn: String!) {
     mlFeature(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLFeature
         privileges {
             ...entityPrivileges

datahub-web-react/src/graphql/mlFeatureTable.graphql

@@ -1,5 +1,7 @@
 query getMLFeatureTable($urn: String!) {
     mlFeatureTable(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLFeatureTable
         privileges {
             ...entityPrivileges

datahub-web-react/src/graphql/mlModel.graphql

@@ -1,5 +1,7 @@
 query getMLModel($urn: String!) {
     mlModel(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLModel
         features: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 100 }) {
             start

datahub-web-react/src/graphql/mlModelGroup.graphql

@@ -1,5 +1,7 @@
 query getMLModelGroup($urn: String!) {
     mlModelGroup(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLModelGroupFields
         incoming: relationships(
             input: {

datahub-web-react/src/graphql/mlPrimaryKey.graphql

@@ -1,5 +1,7 @@
 query getMLPrimaryKey($urn: String!) {
     mlPrimaryKey(urn: $urn) {
+        urn
+        type
         ...nonRecursiveMLPrimaryKey
         privileges {
             ...entityPrivileges

datahub-web-react/src/graphql/tag.graphql

@@ -1,6 +1,7 @@
 query getTag($urn: String!) {
     tag(urn: $urn) {
         urn
+        type
         name
         description
         properties {

datahub-web-react/src/graphql/user.graphql

@@ -1,6 +1,7 @@
 query getUser($urn: String!, $groupsCount: Int!) {
     corpUser(urn: $urn) {
         urn
+        type
         username
         isNativeUser
         exists

docs/managed-datahub/managed-datahub-overview.md

@@ -51,10 +51,17 @@ know.
 | Feature | DataHub | Acryl DataHub |
 | ---------------------------------------------- | ------- | ------------- |
 | Surface data quality results | ✅ | ✅ |
-| Build and enforce continuous data SLAs | ❌ | ✅ |
-| Continuous monitoring of dataset health | ❌ | ✅ |
-| Data observability alerts and notifications | ❌ | ✅ |
-| Data Incident management | ❌ | ✅ |
+| Create data contracts | ✅ | ✅ |
+| Raise and Resolve Data Incidents | ✅ | ✅ |
+| Monitor Freshness SLAs | ❌ | ✅ |
+| Monitor Table Schemas | ❌ | ✅ |
+| Monitor Table Volume | ❌ | ✅ |
+| Validate Table Columns | ❌ | ✅ |
+| Receive Notifications via Email & Slack | ❌ | ✅ |
+| Manage Data Incidents via Slack | ❌ | ✅ |
+| View Data Health Dashboard | ❌ | ✅ |
+| Evaluate data quality checks on-demand (API + UI) | ❌ | ✅ |
+| Evaluate data quality checks in your VPC | ❌ | ✅ |
 
 
 ## Enterprise Grade

metadata-ingestion/setup.py

@@ -345,6 +345,7 @@
         "flask-openid>=1.3.0",
         "dask[dataframe]<2024.7.0",
     },
+    "grafana": {"requests"},
     "glue": aws_common,
     # hdbcli is supported officially by SAP, sqlalchemy-hana is built on top but not officially supported
     "hana": sql_common
@@ -635,6 +636,7 @@
         "dynamodb = datahub.ingestion.source.dynamodb.dynamodb:DynamoDBSource",
         "elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource",
         "feast = datahub.ingestion.source.feast:FeastRepositorySource",
+        "grafana = datahub.ingestion.source.grafana.grafana_source:GrafanaSource",
         "glue = datahub.ingestion.source.aws.glue:GlueSource",
         "sagemaker = datahub.ingestion.source.aws.sagemaker:SagemakerSource",
         "hana = datahub.ingestion.source.sql.hana:HanaSource",

metadata-ingestion/src/datahub/api/entities/dataset/dataset.py

@@ -162,6 +162,7 @@ class Dataset(BaseModel):
     structured_properties: Optional[
         Dict[str, Union[str, float, List[Union[str, float]]]]
     ] = None
+    external_url: Optional[str] = None
 
     @property
     def platform_urn(self) -> str:
@@ -236,6 +237,7 @@ def generate_mcp(
                 description=self.description,
                 name=self.name,
                 customProperties=self.properties,
+                externalUrl=self.external_url,
             ),
         )
         yield mcp

metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py

@@ -1216,8 +1216,22 @@ def _generate_single_profile(
             except Exception as e:
                 if not self.config.catch_exceptions:
                     raise e
-                logger.exception(f"Encountered exception while profiling {pretty_name}")
-                self.report.report_warning(pretty_name, f"Profiling exception {e}")
+
+                error_message = str(e).lower()
+                if "permission denied" in error_message:
+                    self.report.warning(
+                        title="Unauthorized to extract data profile statistics",
+                        message="We were denied access while attempting to generate profiling statistics for some assets. Please ensure the provided user has permission to query these tables and views.",
+                        context=f"Asset: {pretty_name}",
+                        exc=e,
+                    )
+                else:
+                    self.report.warning(
+                        title="Failed to extract statistics for some assets",
+                        message="Caught unexpected exception while attempting to extract profiling statistics for some assets.",
+                        context=f"Asset: {pretty_name}",
+                        exc=e,
+                    )
                 return None
             finally:
                 if batch is not None and self.base_engine.engine.name == TRINO:

metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py

@@ -0,0 +1,131 @@
+from typing import Iterable, List, Optional
+
+import requests
+from pydantic import Field, SecretStr
+
+import datahub.emitter.mce_builder as builder
+from datahub.configuration.source_common import PlatformInstanceConfigMixin
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import MetadataWorkUnitProcessor
+from datahub.ingestion.api.source_helpers import auto_workunit
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+    StaleEntityRemovalSourceReport,
+    StatefulIngestionConfigBase,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionReport,
+    StatefulIngestionSourceBase,
+)
+from datahub.metadata.com.linkedin.pegasus2avro.common import ChangeAuditStamps
+from datahub.metadata.schema_classes import DashboardInfoClass, StatusClass
+
+
+class GrafanaSourceConfig(StatefulIngestionConfigBase, PlatformInstanceConfigMixin):
+    url: str = Field(
+        default="",
+        description="Grafana URL in the format http://your-grafana-instance with no trailing slash",
+    )
+    service_account_token: SecretStr = Field(
+        description="Service account token for Grafana"
+    )
+
+
+class GrafanaReport(StaleEntityRemovalSourceReport):
+    pass
+
+
+@platform_name("Grafana")
+@config_class(GrafanaSourceConfig)
+@support_status(SupportStatus.TESTING)
+class GrafanaSource(StatefulIngestionSourceBase):
+    """
+    This is an experimental source for Grafana.
+    Currently only ingests dashboards (no charts)
+    """
+
+    def __init__(self, config: GrafanaSourceConfig, ctx: PipelineContext):
+        super().__init__(config, ctx)
+        self.source_config = config
+        self.report = GrafanaReport()
+        self.platform = "grafana"
+
+    @classmethod
+    def create(cls, config_dict, ctx):
+        config = GrafanaSourceConfig.parse_obj(config_dict)
+        return cls(config, ctx)
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            StaleEntityRemovalHandler.create(
+                self, self.source_config, self.ctx
+            ).workunit_processor,
+        ]
+
+    def get_report(self) -> StatefulIngestionReport:
+        return self.report
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        headers = {
+            "Authorization": f"Bearer {self.source_config.service_account_token.get_secret_value()}",
+            "Content-Type": "application/json",
+        }
+        try:
+            response = requests.get(
+                f"{self.source_config.url}/api/search", headers=headers
+            )
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            self.report.report_failure(f"Failed to fetch dashboards: {str(e)}")
+            return
+        res_json = response.json()
+        for item in res_json:
+            uid = item["uid"]
+            title = item["title"]
+            url_path = item["url"]
+            full_url = f"{self.source_config.url}{url_path}"
+            dashboard_urn = builder.make_dashboard_urn(
+                platform=self.platform,
+                name=uid,
+                platform_instance=self.source_config.platform_instance,
+            )
+
+            yield from auto_workunit(
+                MetadataChangeProposalWrapper.construct_many(
+                    entityUrn=dashboard_urn,
+                    aspects=[
+                        DashboardInfoClass(
+                            description="",
+                            title=title,
+                            charts=[],
+                            lastModified=ChangeAuditStamps(),
+                            externalUrl=full_url,
+                            customProperties={
+                                key: str(value)
+                                for key, value in {
+                                    "displayName": title,
+                                    "id": item["id"],
+                                    "uid": uid,
+                                    "title": title,
+                                    "uri": item["uri"],
+                                    "type": item["type"],
+                                    "folderId": item.get("folderId"),
+                                    "folderUid": item.get("folderUid"),
+                                    "folderTitle": item.get("folderTitle"),
+                                }.items()
+                                if value is not None
+                            },
+                        ),
+                        StatusClass(removed=False),
+                    ],
+                )
+            )