feat(ingestion/looker): filter Looker dashboards by folder (#11205)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>

metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py

@@ -298,6 +298,16 @@ class LookerDashboardSourceConfig(
         description="When enabled, platform instance will be added in dashboard and chart urn.",
     )
 
+    folder_path_pattern: AllowDenyPattern = Field(
+        default=AllowDenyPattern.allow_all(),
+        description="Allow or deny dashboards from specific folders. "
+        "For example: \n"
+        "deny: \n"
+        " - sales/deprecated \n"
+        "This pattern will deny the ingestion of all dashboards and looks within the sales/deprecated folder. \n"
+        "Dashboards will only be ingested if they're allowed by both this config and dashboard_pattern.",
+    )
+
     @validator("external_base_url", pre=True, always=True)
     def external_url_defaults_to_api_config_base_url(
         cls, v: Optional[str], *, values: Dict[str, Any], **kwargs: Dict[str, Any]

metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py

@@ -1334,6 +1334,17 @@ def process_dashboard(
         looker_dashboard = self._get_looker_dashboard(dashboard_object)
 
         workunits = []
+        if (
+            looker_dashboard.folder_path is not None
+            and not self.source_config.folder_path_pattern.allowed(
+                looker_dashboard.folder_path
+            )
+        ):
+            logger.debug(
+                f"Folder path {looker_dashboard.folder_path} is denied in folder_path_pattern"
+            )
+            return [], None, dashboard_id, start_time, datetime.datetime.now()
+
         if looker_dashboard.folder:
             workunits += list(
                 self._get_folder_and_ancestors_workunits(looker_dashboard.folder)