Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove non-destructive mode (#1415) #1419

Merged
merged 1 commit into from
Apr 30, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions core/dbt/flags.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
STRICT_MODE = False
NON_DESTRUCTIVE = False
FULL_REFRESH = False
USE_CACHE = True
WARN_ERROR = False
TEST_NEW_PARSER = False


def reset():
global STRICT_MODE, NON_DESTRUCTIVE, FULL_REFRESH, USE_CACHE, WARN_ERROR, \
TEST_NEW_PARSER
global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER

STRICT_MODE = False
NON_DESTRUCTIVE = False
FULL_REFRESH = False
USE_CACHE = True
WARN_ERROR = False
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,16 @@
schema=schema,
database=database, type='table') -%}

{%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
{%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}

{%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
{%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}

{%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%}
{%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%}
{%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%}
{%- set should_drop = (full_refresh_mode or exists_not_as_table) -%}

-- setup
{% if old_relation is none -%}
-- noop
{%- elif should_truncate -%}
{{ adapter.truncate_relation(old_relation) }}
{%- elif should_drop -%}
{{ adapter.drop_relation(old_relation) }}
{%- set old_relation = none -%}
Expand All @@ -49,7 +44,7 @@
{{ run_hooks(pre_hooks, inside_transaction=True) }}

-- build model
{% if force_create or old_relation is none -%}
{% if full_refresh_mode or old_relation is none -%}
{%- call statement('main') -%}
{{ create_table_as(False, target_relation, sql) }}
{%- endcall -%}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
{%- set identifier = model['alias'] -%}
{%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}
{%- set backup_identifier = model['name'] + '__dbt_backup' -%}
{%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}

{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
{%- set target_relation = api.Relation.create(identifier=identifier,
Expand All @@ -24,60 +23,28 @@

{%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
{%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
{%- set create_as_temporary = (exists_as_table and non_destructive_mode) -%}


-- drop the temp relations if they exists for some reason
{{ adapter.drop_relation(intermediate_relation) }}
{{ adapter.drop_relation(backup_relation) }}

-- setup: if the target relation already exists, truncate or drop it (if it's a view)
{% if non_destructive_mode -%}
{% if exists_as_table -%}
{{ adapter.truncate_relation(old_relation) }}
{% elif exists_as_view -%}
{{ adapter.drop_relation(old_relation) }}
{%- set old_relation = none -%}
{%- endif %}
{%- endif %}

{{ run_hooks(pre_hooks, inside_transaction=False) }}

-- `BEGIN` happens here:
{{ run_hooks(pre_hooks, inside_transaction=True) }}

-- build model
{% call statement('main') -%}
{%- if non_destructive_mode -%}
{%- if old_relation is not none -%}
{{ create_table_as(create_as_temporary, intermediate_relation, sql) }}

{% set dest_columns = adapter.get_columns_in_relation(target_relation) %}
{% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %}

insert into {{ target_relation }} ({{ dest_cols_csv }}) (
select {{ dest_cols_csv }}
from {{ intermediate_relation.include(database=(not create_as_temporary),
schema=(not create_as_temporary)) }}
);
{%- else -%}
{{ create_table_as(create_as_temporary, target_relation, sql) }}
{%- endif -%}
{%- else -%}
{{ create_table_as(create_as_temporary, intermediate_relation, sql) }}
{%- endif -%}
{{ create_table_as(False, intermediate_relation, sql) }}
{%- endcall %}

-- cleanup
{% if non_destructive_mode -%}
-- noop
{%- else -%}
{% if old_relation is not none %}
{{ adapter.rename_relation(target_relation, backup_relation) }}
{% endif %}
{% if old_relation is not none %}
{{ adapter.rename_relation(target_relation, backup_relation) }}
{% endif %}

{{ adapter.rename_relation(intermediate_relation, target_relation) }}
{%- endif %}
{{ adapter.rename_relation(intermediate_relation, target_relation) }}

{{ run_hooks(post_hooks, inside_transaction=True) }}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@

{% macro handle_existing_table(full_refresh, non_destructive_mode, old_relation) %}
{{ adapter_macro("dbt.handle_existing_table", full_refresh, non_destructive_mode, old_relation) }}
{% macro handle_existing_table(full_refresh, old_relation) %}
{{ adapter_macro("dbt.handle_existing_table", full_refresh, old_relation) }}
{% endmacro %}

{% macro default__handle_existing_table(full_refresh, non_destructive_mode, old_relation) %}
{%- if not non_destructive_mode -%}
{{ adapter.drop_relation(old_relation) }}
{%- endif -%}
{% macro default__handle_existing_table(full_refresh, old_relation) %}
{{ adapter.drop_relation(old_relation) }}
{% endmacro %}

{# /*
Expand All @@ -23,7 +21,6 @@

{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}
{%- set identifier = model['alias'] -%}
{%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}

{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}

Expand All @@ -33,9 +30,6 @@
identifier=identifier, schema=schema, database=database,
type='view') -%}

{%- set should_ignore = non_destructive_mode and exists_as_view %}
{%- set has_transactional_hooks = (hooks | selectattr('transaction', 'equalto', True) | list | length) > 0 %}

{% if run_outside_transaction_hooks %}
-- no transactions on BigQuery
{{ run_hooks(pre_hooks, inside_transaction=False) }}
Expand All @@ -48,30 +42,17 @@
-- that's an error. If we were told to full refresh, drop it. This behavior differs
-- for Snowflake and BigQuery, so multiple dispatch is used.
{%- if old_relation is not none and old_relation.is_table -%}
{{ handle_existing_table(flags.FULL_REFRESH, non_destructive_mode, old_relation) }}
{{ handle_existing_table(flags.FULL_REFRESH, old_relation) }}
{%- endif -%}

-- build model
{% if non_destructive_mode -%}
{% call noop_statement('main', status="PASS", res=None) -%}
-- Not running : non-destructive mode
{{ sql }}
{%- endcall %}
{%- else -%}
{% call statement('main') -%}
{{ create_view_as(target_relation, sql) }}
{%- endcall %}
{%- endif %}
{% call statement('main') -%}
{{ create_view_as(target_relation, sql) }}
{%- endcall %}

{{ run_hooks(post_hooks, inside_transaction=True) }}

{#
-- Don't commit in non-destructive mode _unless_ there are in-transaction hooks
-- TODO : Figure out some other way of doing this that isn't as fragile
#}
{% if has_transactional_hooks or not should_ignore %}
{{ adapter.commit() }}
{% endif %}
{{ adapter.commit() }}

{% if run_outside_transaction_hooks %}
-- No transactions on BigQuery
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
{%- set identifier = model['alias'] -%}
{%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}
{%- set backup_identifier = model['name'] + '__dbt_backup' -%}
{%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}

{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
{%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,
Expand All @@ -30,9 +29,6 @@

{%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}

{%- set has_transactional_hooks = (hooks | selectattr('transaction', 'equalto', True) | list | length) > 0 %}
{%- set should_ignore = non_destructive_mode and exists_as_view %}

{{ run_hooks(pre_hooks, inside_transaction=False) }}

-- drop the temp relations if they exists for some reason
Expand All @@ -43,45 +39,22 @@
{{ run_hooks(pre_hooks, inside_transaction=True) }}

-- build model
{% if should_ignore -%}
{#
-- Materializations need to a statement with name='main'.
-- We could issue a no-op query here (like `select 1`), but that's wasteful. Instead:
-- 1) write the sql contents out to the compiled dirs
-- 2) return a status and result to the caller
#}
{% call noop_statement('main', status="PASS", res=None) -%}
-- Not running : non-destructive mode
{{ sql }}
{%- endcall %}
{%- else -%}
{% call statement('main') -%}
{{ create_view_as(intermediate_relation, sql) }}
{%- endcall %}
{%- endif %}
{% call statement('main') -%}
{{ create_view_as(intermediate_relation, sql) }}
{%- endcall %}

-- cleanup
{% if not should_ignore -%}
-- move the existing view out of the way
{% if old_relation is not none %}
{{ adapter.rename_relation(target_relation, backup_relation) }}
{% endif %}
{{ adapter.rename_relation(intermediate_relation, target_relation) }}
{%- endif %}
-- move the existing view out of the way
{% if old_relation is not none %}
{{ adapter.rename_relation(target_relation, backup_relation) }}
{% endif %}
{{ adapter.rename_relation(intermediate_relation, target_relation) }}

{{ run_hooks(post_hooks, inside_transaction=True) }}

{#
-- Don't commit in non-destructive mode _unless_ there are in-transaction hooks
-- TODO : Figure out some other way of doing this that isn't as fragile
#}
{% if has_transactional_hooks or not should_ignore %}
{{ adapter.commit() }}
{% endif %}
{{ adapter.commit() }}

{% if not should_ignore %}
{{ drop_relation_if_exists(backup_relation) }}
{% endif %}
{{ drop_relation_if_exists(backup_relation) }}

{{ run_hooks(post_hooks, inside_transaction=False) }}

Expand Down
11 changes: 1 addition & 10 deletions core/dbt/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ def run_from_args(parsed):


def update_flags(parsed):
flags.NON_DESTRUCTIVE = getattr(parsed, 'non_destructive', False)
flags.USE_CACHE = getattr(parsed, 'use_cache', True)

arg_drop_existing = getattr(parsed, 'drop_existing', False)
Expand Down Expand Up @@ -421,14 +420,6 @@ def _add_selection_arguments(*subparsers):

def _add_table_mutability_arguments(*subparsers):
for sub in subparsers:
sub.add_argument(
'--non-destructive',
action='store_true',
help="""
If specified, DBT will not drop views. Tables will be truncated
instead of dropped.
"""
)
sub.add_argument(
'--full-refresh',
action='store_true',
Expand Down Expand Up @@ -675,7 +666,7 @@ def parse_args(args):
# --models, --exclude
_add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub,
archive_sub)
# --full-refresh, --non-destructive
# --full-refresh
_add_table_mutability_arguments(run_sub, compile_sub)

_build_seed_subparser(subs, base_subparser)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,8 @@

{%- set unique_key = config.get('unique_key') -%}

{%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
{%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}

{% if non_destructive_mode %}
{{ exceptions.raise_compiler_error("--non-destructive mode is not supported on BigQuery") }}
{% endif %}

{%- set identifier = model['alias'] -%}

{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
{% materialization table, adapter='bigquery' -%}

{%- set identifier = model['alias'] -%}
{%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%}
{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
{%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}
{%- set target_relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

{% macro bigquery__handle_existing_table(full_refresh, non_destructive_mode, old_relation) %}
{%- if full_refresh and not non_destructive_mode -%}
{% macro bigquery__handle_existing_table(full_refresh, old_relation) %}
{%- if full_refresh -%}
{{ adapter.drop_relation(old_relation) }}
{%- else -%}
{{ exceptions.relation_wrong_type(old_relation, 'view') }}
Expand Down
Loading