From fc99be8744b0ea39e4707c05791fe66759df0436 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 16 Jul 2024 16:15:39 -0700 Subject: [PATCH] fix(ingest/redshift): reduce severity of SQL parsing issues --- .../src/datahub/ingestion/source/redshift/lineage_v2.py | 2 +- .../src/datahub/sql_parsing/sql_parsing_aggregator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py index 3d6c746183fd92..bd0bbe742a2193 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py @@ -419,7 +419,7 @@ def generate(self) -> Iterable[MetadataWorkUnit]: for mcp in self.aggregator.gen_metadata(): yield mcp.as_workunit() if len(self.aggregator.report.observed_query_parse_failures) > 0: - self.report.report_failure( + self.report.report_warning( title="Failed to extract some SQL lineage", message="Unexpected error(s) while attempting to extract lineage from SQL queries. See the full logs for more details.", context=f"Query Parsing Failures: {self.aggregator.report.observed_query_parse_failures}", diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 677b96269fe586..894f01820718b2 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -660,10 +660,10 @@ def add_observed_query( if parsed.debug_info.table_error: self.report.num_observed_queries_failed += 1 return # we can't do anything with this query - elif isinstance(parsed.debug_info.column_error, CooperativeTimeoutError): - self.report.num_observed_queries_column_timeout += 1 elif parsed.debug_info.column_error: self.report.num_observed_queries_column_failed += 1 + if isinstance(parsed.debug_info.column_error, CooperativeTimeoutError): + self.report.num_observed_queries_column_timeout += 1 query_fingerprint = parsed.query_fingerprint