Skip to content

Commit

Permalink
approach 1
Browse files Browse the repository at this point in the history
  • Loading branch information
MichelleArk committed May 13, 2024
1 parent c542fd1 commit 29e533f
Showing 1 changed file with 26 additions and 0 deletions.
26 changes: 26 additions & 0 deletions dbt/adapters/bigquery/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ class BigQueryAdapter(BaseAdapter):
{
Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full),
Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full),
Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full),
}
)

Expand Down Expand Up @@ -721,6 +722,31 @@ def _get_catalog_schemas(self, relation_config: Iterable[RelationConfig]) -> Sch
)
return result

def calculate_freshness_from_metadata_batch(
self,
sources: List[BaseRelation],
macro_resolver: Optional[MacroResolverProtocol] = None,
) -> Tuple[List[Optional[AdapterResponse]], Dict[BaseRelation, FreshnessResponse]]:
freshness_responses: Dict[BaseRelation, FreshnessResponse] = {}
adapter_responses: List[Optional[AdapterResponse]] = []

conn = self.connections.get_thread_connection()
client: google.cloud.bigquery.Client = conn.handle
for source in sources:
table_ref = self.get_table_ref_from_relation(source)
table = client.get_table(table_ref)
snapshot = datetime.now(tz=pytz.UTC)

freshness = FreshnessResponse(
max_loaded_at=table.modified,
snapshotted_at=snapshot,
age=(snapshot - table.modified).total_seconds(),
)
freshness_responses[source] = freshness
adapter_responses.append(None)

return adapter_responses, freshness_responses

def calculate_freshness_from_metadata(
self,
source: BaseRelation,
Expand Down

0 comments on commit 29e533f

Please sign in to comment.