From 1b0d63515292e3b98cc96c8582b3aaf404185586 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 4 Dec 2019 10:31:07 -0700 Subject: [PATCH] add a formal search order for materializations, tests --- core/dbt/adapters/base/impl.py | 80 +++++++------ core/dbt/contracts/graph/manifest.py | 111 ++++++++++++++---- core/dbt/node_runners.py | 1 + .../models/model.sql | 2 + .../override-view-adapter-dep/dbt_project.yml | 3 + .../macros/override_view.sql | 69 +++++++++++ .../override_view.sql | 69 +++++++++++ .../dbt_project.yml | 3 + .../macros/override_view.sql | 65 ++++++++++ .../override-view-default-dep/dbt_project.yml | 3 + .../macros/default_view.sql | 3 + .../default_view.sql | 3 + .../test_custom_materialization.py | 99 ++++++++++++++++ 13 files changed, 451 insertions(+), 60 deletions(-) create mode 100644 test/integration/053_custom_materialization/models/model.sql create mode 100644 test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml create mode 100644 test/integration/053_custom_materialization/override-view-adapter-dep/macros/override_view.sql create mode 100644 test/integration/053_custom_materialization/override-view-adapter-macros/override_view.sql create mode 100644 test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml create mode 100644 test/integration/053_custom_materialization/override-view-adapter-pass-dep/macros/override_view.sql create mode 100644 test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml create mode 100644 test/integration/053_custom_materialization/override-view-default-dep/macros/default_view.sql create mode 100644 test/integration/053_custom_materialization/override-view-default-macros/default_view.sql create mode 100644 test/integration/053_custom_materialization/test_custom_materialization.py diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 15b534cb5aa..f20d73464e7 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -9,7 +9,11 @@ import agate import pytz -import dbt.exceptions +from dbt.exceptions import ( + raise_database_error, raise_compiler_error, invalid_type_error, + get_relation_returned_multiple_results, + InternalException, NotImplementedException, RuntimeException, +) import dbt.flags from dbt import deprecations @@ -37,7 +41,7 @@ def _expect_row_value(key: str, row: agate.Row): if key not in row.keys(): - raise dbt.exceptions.InternalException( + raise InternalException( 'Got a row without "{}" column, columns: {}' .format(key, row.keys()) ) @@ -84,14 +88,14 @@ def _utc( assume the datetime is already for UTC and add the timezone. """ if dt is None: - raise dbt.exceptions.raise_database_error( + raise raise_database_error( "Expected a non-null value when querying field '{}' of table " " {} but received value 'null' instead".format( field_name, source)) elif not hasattr(dt, 'tzinfo'): - raise dbt.exceptions.raise_database_error( + raise raise_database_error( "Expected a timestamp value when querying field '{}' of table " "{} but received value of type '{}' instead".format( field_name, @@ -141,7 +145,7 @@ def flatten(self): # make sure we don't have duplicates seen = {r.database.lower() for r in self if r.database} if len(seen) > 1: - dbt.exceptions.raise_compiler_error(str(seen)) + raise_compiler_error(str(seen)) for information_schema_name, schema in self.search(): path = { @@ -380,7 +384,7 @@ def cache_added(self, relation: Optional[BaseRelation]) -> str: """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: name = self.nice_connection_name() - dbt.exceptions.raise_compiler_error( + raise_compiler_error( 'Attempted to cache a null relation for {}'.format(name) ) if dbt.flags.USE_CACHE: @@ -395,7 +399,7 @@ def cache_dropped(self, relation: Optional[BaseRelation]) -> str: """ if relation is None: name = self.nice_connection_name() - dbt.exceptions.raise_compiler_error( + raise_compiler_error( 'Attempted to drop a null relation for {}'.format(name) ) if dbt.flags.USE_CACHE: @@ -415,7 +419,7 @@ def cache_renamed( name = self.nice_connection_name() src_name = _relation_name(from_relation) dst_name = _relation_name(to_relation) - dbt.exceptions.raise_compiler_error( + raise_compiler_error( 'Attempted to rename {} to {} for {}' .format(src_name, dst_name, name) ) @@ -430,12 +434,12 @@ def cache_renamed( @abc.abstractclassmethod def date_function(cls) -> str: """Get the date function used by this adapter's database.""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`date_function` is not implemented for this adapter!') @abc.abstractclassmethod def is_cancelable(cls) -> bool: - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`is_cancelable` is not implemented for this adapter!' ) @@ -445,7 +449,7 @@ def is_cancelable(cls) -> bool: @abc.abstractmethod def list_schemas(self, database: str) -> List[str]: """Get a list of existing schemas in database""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`list_schemas` is not implemented for this adapter!' ) @@ -473,7 +477,7 @@ def drop_relation(self, relation: BaseRelation) -> None: *Implementors must call self.cache.drop() to preserve cache state!* """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`drop_relation` is not implemented for this adapter!' ) @@ -481,7 +485,7 @@ def drop_relation(self, relation: BaseRelation) -> None: @available.parse_none def truncate_relation(self, relation: BaseRelation) -> None: """Truncate the given relation.""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`truncate_relation` is not implemented for this adapter!' ) @@ -494,7 +498,7 @@ def rename_relation( Implementors must call self.cache.rename() to preserve cache state. """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`rename_relation` is not implemented for this adapter!' ) @@ -504,7 +508,7 @@ def get_columns_in_relation( self, relation: BaseRelation ) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`get_columns_in_relation` is not implemented for this adapter!' ) @@ -532,7 +536,7 @@ def expand_column_types( :param self.Relation current: A relation that currently exists in the database with columns of unspecified types. """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`expand_target_column_types` is not implemented for this adapter!' ) @@ -550,7 +554,7 @@ def list_relations_without_caching( :return: The relations in schema :rtype: List[self.Relation] """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`list_relations_without_caching` is not implemented for this ' 'adapter!' ) @@ -566,14 +570,14 @@ def get_missing_columns( to_relation. """ if not isinstance(from_relation, self.Relation): - dbt.exceptions.invalid_type_error( + invalid_type_error( method_name='get_missing_columns', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): - dbt.exceptions.invalid_type_error( + invalid_type_error( method_name='get_missing_columns', arg_name='to_relation', got_value=to_relation, @@ -602,11 +606,11 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: expected columns. :param Relation relation: The relation to check - :raises dbt.exceptions.CompilationException: If the columns are + :raises CompilationException: If the columns are incorrect. """ if not isinstance(relation, self.Relation): - dbt.exceptions.invalid_type_error( + invalid_type_error( method_name='valid_snapshot_target', arg_name='relation', got_value=relation, @@ -636,21 +640,21 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: 'Snapshot target is not a snapshot table (missing "{}")' .format('", "'.join(missing)) ) - dbt.exceptions.raise_compiler_error(msg) + raise_compiler_error(msg) @available.parse_none def expand_target_column_types( self, from_relation: BaseRelation, to_relation: BaseRelation ) -> None: if not isinstance(from_relation, self.Relation): - dbt.exceptions.invalid_type_error( + invalid_type_error( method_name='expand_target_column_types', arg_name='from_relation', got_value=from_relation, expected_type=self.Relation) if not isinstance(to_relation, self.Relation): - dbt.exceptions.invalid_type_error( + invalid_type_error( method_name='expand_target_column_types', arg_name='to_relation', got_value=to_relation, @@ -731,7 +735,7 @@ def get_relation( 'schema': schema, 'database': database, } - dbt.exceptions.get_relation_returned_multiple_results( + get_relation_returned_multiple_results( kwargs, matches ) @@ -755,14 +759,14 @@ def already_exists(self, schema: str, name: str) -> bool: @available.parse_none def create_schema(self, database: str, schema: str): """Create the given schema if it does not exist.""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`create_schema` is not implemented for this adapter!' ) @abc.abstractmethod def drop_schema(self, database: str, schema: str): """Drop the given schema (and everything in it) if it exists.""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`drop_schema` is not implemented for this adapter!' ) @@ -770,7 +774,7 @@ def drop_schema(self, database: str, schema: str): @abc.abstractclassmethod def quote(cls, identifier: str) -> str: """Quote the given identifier, as appropriate for the database.""" - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`quote` is not implemented for this adapter!' ) @@ -804,7 +808,7 @@ def quote_seed_column( elif quote_config is None: deprecations.warn('column-quoting-unset') else: - dbt.exceptions.raise_compiler_error( + raise_compiler_error( f'The seed configuration value of "quote_columns" has an ' f'invalid type {type(quote_config)}' ) @@ -829,7 +833,7 @@ def convert_text_type( :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`convert_text_type` is not implemented for this adapter!') @abc.abstractclassmethod @@ -843,7 +847,7 @@ def convert_number_type( :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`convert_number_type` is not implemented for this adapter!') @abc.abstractclassmethod @@ -857,7 +861,7 @@ def convert_boolean_type( :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`convert_boolean_type` is not implemented for this adapter!') @abc.abstractclassmethod @@ -871,7 +875,7 @@ def convert_datetime_type( :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`convert_datetime_type` is not implemented for this adapter!') @abc.abstractclassmethod @@ -883,7 +887,7 @@ def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str: :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`convert_date_type` is not implemented for this adapter!') @abc.abstractclassmethod @@ -895,7 +899,7 @@ def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str: :param col_idx: The index into the agate table for the column. :return: The name of the type in the database """ - raise dbt.exceptions.NotImplementedException( + raise NotImplementedException( '`convert_time_type` is not implemented for this adapter!') @available @@ -958,9 +962,7 @@ def execute_macro( else: package_name = 'the "{}" package'.format(project) - # The import of dbt.context.runtime below shadows 'dbt' - import dbt.exceptions - raise dbt.exceptions.RuntimeException( + raise RuntimeException( 'dbt could not find a macro with the name "{}" in {}' .format(macro_name, package_name) ) @@ -1034,7 +1036,7 @@ def calculate_freshness( # now we have a 1-row table of the maximum `loaded_at_field` value and # the current time according to the db. if len(table) != 1 or len(table[0]) != 2: - dbt.exceptions.raise_compiler_error( + raise_compiler_error( 'Got an invalid result from "{}" macro: {}'.format( FRESHNESS_MACRO_NAME, [tuple(r) for r in table] ) diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 81a0e9bc9dd..3ee4b52a4dc 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -1,3 +1,4 @@ +import enum import hashlib import os from dataclasses import dataclass, field @@ -11,7 +12,10 @@ ParsedDocumentation from dbt.contracts.graph.compiled import CompileResultNode from dbt.contracts.util import Writable, Replaceable -from dbt.exceptions import raise_duplicate_resource_name, InternalException +from dbt.exceptions import ( + raise_duplicate_resource_name, InternalException, raise_compiler_error +) +from dbt.include.global_project import PACKAGES from dbt.logger import GLOBAL_LOGGER as logger from dbt.node_types import NodeType from dbt import tracking @@ -208,6 +212,50 @@ def _deepcopy(value): return value.from_dict(value.to_dict()) +class Locality(enum.IntEnum): + Core = 1 + Imported = 2 + Root = 3 + + +class Specificity(enum.IntEnum): + Default = 1 + Adapter = 2 + + +@dataclass +class MaterializationCandidate: + specificity: Specificity + locality: Locality + macro: ParsedMacro + + def __eq__(self, other): + equal = ( + self.specificity == other.specificity and + self.locality == other.locality + ) + if equal: + raise_compiler_error( + 'Found two materializations with the name {} (packages {} and ' + '{}). dbt cannot resolve this ambiguity' + .format(self.macro.name, self.macro.package_name, + other.macro.package_name) + ) + + return equal + + def __lt__(self, other: 'MaterializationCandidate') -> bool: + if self.specificity < other.specificity: + return True + if self.specificity > other.specificity: + return False + if self.locality < other.locality: + return True + if self.locality > other.locality: + return False + return False + + @dataclass class Manifest: """The manifest for the full graph, after parsing and during compilation. @@ -291,7 +339,7 @@ def find_docs_by_name(self, name, package=None): parts = unique_id.split('.') if len(parts) != 2: msg = "documentation names cannot contain '.' characters" - dbt.exceptions.raise_compiler_error(msg, doc) + raise_compiler_error(msg, doc) found_package, found_node = parts @@ -318,27 +366,48 @@ def find_source_by_name(self, source_name, table_name, package): name = '{}.{}'.format(source_name, table_name) return self._find_by_name(name, package, 'nodes', [NodeType.Source]) - def get_materialization_macro(self, materialization_name, - adapter_type=None): - macro_name = dbt.utils.get_materialization_macro_name( - materialization_name=materialization_name, - adapter_type=adapter_type, - with_prefix=False) - - macro = self.find_macro_by_name( - macro_name, - None) - - if adapter_type not in ('default', None) and macro is None: - macro_name = dbt.utils.get_materialization_macro_name( + def get_materialization_macro( + self, project_name: str, materialization_name: str, adapter_type: str + ): + adapter_macro_name, default_macro_name = [ + dbt.utils.get_materialization_macro_name( materialization_name=materialization_name, - adapter_type='default', - with_prefix=False) - macro = self.find_macro_by_name( - macro_name, - None) + adapter_type=atype, + with_prefix=False, + ) + for atype in (adapter_type, None) + ] + + candidates: List[MaterializationCandidate] = [] + + for unique_id, macro in self.macros.items(): + specificity: Specificity + locality: Locality + if macro.name == adapter_macro_name: + specificity = Specificity.Adapter + elif macro.name == default_macro_name: + specificity = Specificity.Default + else: + continue - return macro + if macro.package_name == project_name: + locality = Locality.Root + elif macro.package_name in PACKAGES: + locality = Locality.Core + else: + locality = Locality.Imported + + candidate = MaterializationCandidate( + specificity=specificity, + locality=locality, + macro=macro, + ) + candidates.append(candidate) + + if not candidates: + return None + candidates.sort() + return candidates[-1].macro def get_resource_fqns(self): resource_fqns = {} diff --git a/core/dbt/node_runners.py b/core/dbt/node_runners.py index c5c56094996..23951f68426 100644 --- a/core/dbt/node_runners.py +++ b/core/dbt/node_runners.py @@ -417,6 +417,7 @@ def execute(self, model, manifest): model, self.config, manifest) materialization_macro = manifest.get_materialization_macro( + self.config.project_name, model.get_materialization(), self.adapter.type()) diff --git a/test/integration/053_custom_materialization/models/model.sql b/test/integration/053_custom_materialization/models/model.sql new file mode 100644 index 00000000000..0217b87a38a --- /dev/null +++ b/test/integration/053_custom_materialization/models/model.sql @@ -0,0 +1,2 @@ +{{ config(materialized='view') }} +select 1 as id diff --git a/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml b/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml new file mode 100644 index 00000000000..2c58789b48e --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml @@ -0,0 +1,3 @@ +name: view_adapter_override +version: '1.0' +macro-paths: ['macros'] diff --git a/test/integration/053_custom_materialization/override-view-adapter-dep/macros/override_view.sql b/test/integration/053_custom_materialization/override-view-adapter-dep/macros/override_view.sql new file mode 100644 index 00000000000..4fe3ece9052 --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-adapter-dep/macros/override_view.sql @@ -0,0 +1,69 @@ +{%- materialization view, adapter='postgres' -%} +{{ exceptions.raise_compiler_error('intentionally raising an error in the postgres view materialization') }} +{%- endmaterialization -%} + +{# copy+pasting the default view impl #} +{% materialization view, default %} + + {%- set identifier = model['alias'] -%} + {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} + {%- set backup_identifier = model['name'] + '__dbt_backup' -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, + type='view') -%} + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, + schema=schema, database=database, type='view') -%} + + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "old_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the old_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the old_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} + {%- set backup_relation = api.Relation.create(identifier=backup_identifier, + schema=schema, database=database, + type=backup_relation_type) -%} + + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exists for some reason + {{ adapter.drop_relation(intermediate_relation) }} + {{ adapter.drop_relation(backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ create_view_as(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if old_relation is not none %} + {{ adapter.rename_relation(target_relation, backup_relation) }} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/test/integration/053_custom_materialization/override-view-adapter-macros/override_view.sql b/test/integration/053_custom_materialization/override-view-adapter-macros/override_view.sql new file mode 100644 index 00000000000..4fe3ece9052 --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-adapter-macros/override_view.sql @@ -0,0 +1,69 @@ +{%- materialization view, adapter='postgres' -%} +{{ exceptions.raise_compiler_error('intentionally raising an error in the postgres view materialization') }} +{%- endmaterialization -%} + +{# copy+pasting the default view impl #} +{% materialization view, default %} + + {%- set identifier = model['alias'] -%} + {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} + {%- set backup_identifier = model['name'] + '__dbt_backup' -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, + type='view') -%} + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, + schema=schema, database=database, type='view') -%} + + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "old_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the old_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the old_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} + {%- set backup_relation = api.Relation.create(identifier=backup_identifier, + schema=schema, database=database, + type=backup_relation_type) -%} + + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exists for some reason + {{ adapter.drop_relation(intermediate_relation) }} + {{ adapter.drop_relation(backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ create_view_as(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if old_relation is not none %} + {{ adapter.rename_relation(target_relation, backup_relation) }} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml b/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml new file mode 100644 index 00000000000..2c58789b48e --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml @@ -0,0 +1,3 @@ +name: view_adapter_override +version: '1.0' +macro-paths: ['macros'] diff --git a/test/integration/053_custom_materialization/override-view-adapter-pass-dep/macros/override_view.sql b/test/integration/053_custom_materialization/override-view-adapter-pass-dep/macros/override_view.sql new file mode 100644 index 00000000000..896c1df0112 --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-adapter-pass-dep/macros/override_view.sql @@ -0,0 +1,65 @@ +{# copy+pasting the default view impl #} +{% materialization view, default %} + + {%- set identifier = model['alias'] -%} + {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%} + {%- set backup_identifier = model['name'] + '__dbt_backup' -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database, + type='view') -%} + {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier, + schema=schema, database=database, type='view') -%} + + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "old_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the old_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the old_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%} + {%- set backup_relation = api.Relation.create(identifier=backup_identifier, + schema=schema, database=database, + type=backup_relation_type) -%} + + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exists for some reason + {{ adapter.drop_relation(intermediate_relation) }} + {{ adapter.drop_relation(backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ create_view_as(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if old_relation is not none %} + {{ adapter.rename_relation(target_relation, backup_relation) }} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml b/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml new file mode 100644 index 00000000000..9b1515079d2 --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml @@ -0,0 +1,3 @@ +name: view_default_override +version: '1.0' +macro-paths: ['macros'] diff --git a/test/integration/053_custom_materialization/override-view-default-dep/macros/default_view.sql b/test/integration/053_custom_materialization/override-view-default-dep/macros/default_view.sql new file mode 100644 index 00000000000..536d0cd4591 --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-default-dep/macros/default_view.sql @@ -0,0 +1,3 @@ +{%- materialization view, default -%} +{{ exceptions.raise_compiler_error('intentionally raising an error in the default view materialization') }} +{%- endmaterialization -%} diff --git a/test/integration/053_custom_materialization/override-view-default-macros/default_view.sql b/test/integration/053_custom_materialization/override-view-default-macros/default_view.sql new file mode 100644 index 00000000000..536d0cd4591 --- /dev/null +++ b/test/integration/053_custom_materialization/override-view-default-macros/default_view.sql @@ -0,0 +1,3 @@ +{%- materialization view, default -%} +{{ exceptions.raise_compiler_error('intentionally raising an error in the default view materialization') }} +{%- endmaterialization -%} diff --git a/test/integration/053_custom_materialization/test_custom_materialization.py b/test/integration/053_custom_materialization/test_custom_materialization.py new file mode 100644 index 00000000000..c594e69699e --- /dev/null +++ b/test/integration/053_custom_materialization/test_custom_materialization.py @@ -0,0 +1,99 @@ +from test.integration.base import DBTIntegrationTest, use_profile +import os + + +class BaseTestCustomMaterialization(DBTIntegrationTest): + @property + def schema(self): + return 'dbt_custom_materializations_053' + + @staticmethod + def dir(value): + return os.path.normpath(value) + + @property + def models(self): + return "models" + + +class TestOverrideAdapterDependency(BaseTestCustomMaterialization): + # make sure that if there's a dependency with an adapter-specific + # materialization, we honor that materialization + @property + def packages_config(self): + return { + 'packages': [ + { + 'local': 'override-view-adapter-dep' + } + ] + } + + @use_profile('postgres') + def test_postgres_adapter_dependency(self): + self.run_dbt(['deps']) + # this should error because the override is buggy + self.run_dbt(['run'], expect_pass=False) + + +class TestOverrideDefaultDependency(BaseTestCustomMaterialization): + @property + def packages_config(self): + return { + 'packages': [ + { + 'local': 'override-view-default-dep' + } + ] + } + + @use_profile('postgres') + def test_postgres_default_dependency(self): + self.run_dbt(['deps']) + # this should error because the override is buggy + self.run_dbt(['run'], expect_pass=False) + + +class TestOverrideAdapterDependencyPassing(BaseTestCustomMaterialization): + @property + def packages_config(self): + return { + 'packages': [ + { + 'local': 'override-view-adapter-pass-dep' + } + ] + } + + @use_profile('postgres') + def test_postgres_default_dependency(self): + self.run_dbt(['deps']) + # this should pass because the override is ok + self.run_dbt(['run']) + + +class TestOverrideAdapterLocal(BaseTestCustomMaterialization): + # make sure that the local default wins over the dependency + # adapter-specific + + @property + def packages_config(self): + return { + 'packages': [ + { + 'local': 'override-view-adapter-pass-dep' + } + ] + } + + @property + def project_config(self): + return { + 'macro-paths': ['override-view-adapter-macros'] + } + + @use_profile('postgres') + def test_postgres_default_dependency(self): + self.run_dbt(['deps']) + # this should error because the override is buggy + self.run_dbt(['run'], expect_pass=False)