From 9bdf5fe74aca9d3366be594c498332cd8ae2c1f1 Mon Sep 17 00:00:00 2001 From: Nathaniel May Date: Thu, 2 Dec 2021 18:35:51 -0500 Subject: [PATCH] use reference keys instead of relations (#4410) --- core/dbt/adapters/base/impl.py | 8 ++++++-- core/dbt/adapters/cache.py | 16 ++-------------- core/dbt/adapters/reference_keys.py | 24 ++++++++++++++++++++++++ core/dbt/adapters/sql/impl.py | 5 +++-- core/dbt/events/types.py | 24 ++++++++---------------- core/dbt/task/deps.py | 2 +- test/unit/test_events.py | 6 +++--- 7 files changed, 47 insertions(+), 38 deletions(-) create mode 100644 core/dbt/adapters/reference_keys.py diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index deeb06d84dd..d55de305d7a 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -39,7 +39,7 @@ ComponentName, BaseRelation, InformationSchema, SchemaSearchMap ) from dbt.adapters.base import Column as BaseColumn -from dbt.adapters.cache import RelationsCache +from dbt.adapters.cache import RelationsCache, _make_key SeedModel = Union[ParsedSeedNode, CompiledSeedNode] @@ -676,7 +676,11 @@ def list_relations( relations = self.list_relations_without_caching( schema_relation ) - fire_event(ListRelations(database=database, schema=schema, relations=relations)) + fire_event(ListRelations( + database=database, + schema=schema, + relations=[_make_key(x) for x in relations] + )) return relations diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 1f0c9f4ccf9..6c14c0fdb42 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -1,8 +1,8 @@ import threading -from collections import namedtuple from copy import deepcopy from typing import Any, Dict, Iterable, List, Optional, Set, Tuple +from dbt.adapters.reference_keys import _make_key, _ReferenceKey import dbt.exceptions from dbt.events.functions import fire_event from dbt.events.types import ( @@ -22,18 +22,6 @@ ) from dbt.utils import lowercase -_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier') - - -def _make_key(relation) -> _ReferenceKey: - """Make _ReferenceKeys with lowercase values for the cache so we don't have - to keep track of quoting - """ - # databases and schemas can both be None - return _ReferenceKey(lowercase(relation.database), - lowercase(relation.schema), - lowercase(relation.identifier)) - def dot_separated(key: _ReferenceKey) -> str: """Return the key in dot-separated string form. @@ -334,7 +322,7 @@ def add(self, relation): :param BaseRelation relation: The underlying relation. """ cached = _CachedRelation(relation) - fire_event(AddRelation(relation=cached)) + fire_event(AddRelation(relation=_make_key(cached))) fire_event(DumpBeforeAddGraph(dump=self.dump_graph())) with self.lock: diff --git a/core/dbt/adapters/reference_keys.py b/core/dbt/adapters/reference_keys.py new file mode 100644 index 00000000000..5780e0d0beb --- /dev/null +++ b/core/dbt/adapters/reference_keys.py @@ -0,0 +1,24 @@ +# this module exists to resolve circular imports with the events module + +from collections import namedtuple +from typing import Optional + + +_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier') + + +def lowercase(value: Optional[str]) -> Optional[str]: + if value is None: + return None + else: + return value.lower() + + +def _make_key(relation) -> _ReferenceKey: + """Make _ReferenceKeys with lowercase values for the cache so we don't have + to keep track of quoting + """ + # databases and schemas can both be None + return _ReferenceKey(lowercase(relation.database), + lowercase(relation.schema), + lowercase(relation.identifier)) diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index 5dc54f3a6b6..24d97dd137c 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -5,6 +5,7 @@ from dbt.contracts.connection import Connection import dbt.exceptions from dbt.adapters.base import BaseAdapter, available +from dbt.adapters.cache import _make_key from dbt.adapters.sql import SQLConnectionManager from dbt.events.functions import fire_event from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop @@ -182,7 +183,7 @@ def get_columns_in_relation(self, relation): def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() - fire_event(SchemaCreation(relation=relation)) + fire_event(SchemaCreation(relation=_make_key(relation))) kwargs = { 'relation': relation, } @@ -193,7 +194,7 @@ def create_schema(self, relation: BaseRelation) -> None: def drop_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() - fire_event(SchemaDrop(relation=relation)) + fire_event(SchemaDrop(relation=_make_key(relation))) kwargs = { 'relation': relation, } diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 2dbc60a4af6..4ed9ff10c63 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1,11 +1,11 @@ import argparse from dataclasses import dataclass +from dbt.adapters.reference_keys import _make_key, _ReferenceKey from dbt.events.stubs import ( _CachedRelation, BaseRelation, ParsedModelNode, ParsedHookNode, - _ReferenceKey, RunResult ) from dbt import ui @@ -506,7 +506,7 @@ def message(self) -> str: class ListRelations(DebugLevel, Cli, File): database: Optional[str] schema: str - relations: List[BaseRelation] + relations: List[_ReferenceKey] code: str = "E014" def message(self) -> str: @@ -573,20 +573,16 @@ def message(self) -> str: @dataclass class SchemaCreation(DebugLevel, Cli, File): - relation: BaseRelation + relation: _ReferenceKey code: str = "E020" def message(self) -> str: return f'Creating schema "{self.relation}"' - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - @dataclass class SchemaDrop(DebugLevel, Cli, File): - relation: BaseRelation + relation: _ReferenceKey code: str = "E021" def message(self) -> str: @@ -625,16 +621,12 @@ def message(self) -> str: @dataclass class AddRelation(DebugLevel, Cli, File, Cache): - relation: _CachedRelation + relation: _ReferenceKey code: str = "E024" def message(self) -> str: return f"Adding relation: {str(self.relation)}" - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - @dataclass class DropMissingRelation(DebugLevel, Cli, File, Cache): @@ -2499,8 +2491,8 @@ def message(self) -> str: SQLQueryStatus(status="", elapsed=0.1) SQLCommit(conn_name="") ColTypeChange(orig_type="", new_type="", table="") - SchemaCreation(relation=BaseRelation()) - SchemaDrop(relation=BaseRelation()) + SchemaCreation(relation=_make_key(BaseRelation())) + SchemaDrop(relation=_make_key(BaseRelation())) UncachedRelation( dep_key=_ReferenceKey(database="", schema="", identifier=""), ref_key=_ReferenceKey(database="", schema="", identifier=""), @@ -2509,7 +2501,7 @@ def message(self) -> str: dep_key=_ReferenceKey(database="", schema="", identifier=""), ref_key=_ReferenceKey(database="", schema="", identifier=""), ) - AddRelation(relation=_CachedRelation()) + AddRelation(relation=_make_key(_CachedRelation())) DropMissingRelation(relation=_ReferenceKey(database="", schema="", identifier="")) DropCascade( dropped=_ReferenceKey(database="", schema="", identifier=""), diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 3f2a86f033a..8cbab91e398 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -63,7 +63,7 @@ def run(self): source_type = package.source_type() version = package.get_version() - fire_event(DepsStartPackageInstall(package=package)) + fire_event(DepsStartPackageInstall(package=package.nice_version_name())) package.install(self.config, renderer) fire_event(DepsInstallInfo(version_name=package.nice_version_name())) if source_type == 'hub': diff --git a/test/unit/test_events.py b/test/unit/test_events.py index 871df051b83..cfcfae5612a 100644 --- a/test/unit/test_events.py +++ b/test/unit/test_events.py @@ -196,8 +196,8 @@ def MockNode(): SQLQueryStatus(status="", elapsed=0.1), SQLCommit(conn_name=""), ColTypeChange(orig_type="", new_type="", table=""), - SchemaCreation(relation=BaseRelation()), - SchemaDrop(relation=BaseRelation()), + SchemaCreation(relation=_ReferenceKey(database="", schema="", identifier="")), + SchemaDrop(relation=_ReferenceKey(database="", schema="", identifier="")), UncachedRelation( dep_key=_ReferenceKey(database="", schema="", identifier=""), ref_key=_ReferenceKey(database="", schema="", identifier=""), @@ -206,7 +206,7 @@ def MockNode(): dep_key=_ReferenceKey(database="", schema="", identifier=""), ref_key=_ReferenceKey(database="", schema="", identifier=""), ), - AddRelation(relation=_CachedRelation()), + AddRelation(relation=_ReferenceKey(database="", schema="", identifier="")), DropMissingRelation(relation=_ReferenceKey(database="", schema="", identifier="")), DropCascade( dropped=_ReferenceKey(database="", schema="", identifier=""),