From 937219dd917f94b31979f22c88cbc4441415ad83 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 4 Dec 2018 14:11:23 -0700 Subject: [PATCH 01/12] add connection_info method to credentials for dbt debug --- dbt/contracts/connection.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/dbt/contracts/connection.py b/dbt/contracts/connection.py index d5c09a187d0..911b2f13a40 100644 --- a/dbt/contracts/connection.py +++ b/dbt/contracts/connection.py @@ -196,6 +196,19 @@ def type(self): 'type not implemented for base credentials class' ) + def connection_info(self): + """Return an ordered iterator of key/value pairs for pretty-printing. + """ + for key in self._connection_keys(): + if key in self._contents: + yield key, self._contents[key] + + def _connection_keys(self): + """The credential object keys that should be printed to users in + 'dbt debug' output. This is specific to each adapter. + """ + raise NotImplementedError + class PostgresCredentials(Credentials): SCHEMA = POSTGRES_CREDENTIALS_CONTRACT @@ -214,6 +227,9 @@ def password(self): # we can't access this as 'pass' since that's reserved return self._contents['pass'] + def _connection_keys(self): + return ('host', 'port', 'user', 'dbname', 'schema') + class RedshiftCredentials(PostgresCredentials): SCHEMA = REDSHIFT_CREDENTIALS_CONTRACT @@ -226,6 +242,9 @@ def __init__(self, *args, **kwargs): def type(self): return 'redshift' + def _connection_keys(self): + return ('host', 'port', 'user', 'dbname', 'schema', 'method') + class SnowflakeCredentials(Credentials): SCHEMA = SNOWFLAKE_CREDENTIALS_CONTRACT @@ -234,6 +253,9 @@ class SnowflakeCredentials(Credentials): def type(self): return 'snowflake' + def _connection_keys(self): + return ('account', 'user', 'database', 'schema', 'warehouse', 'role') + class BigQueryCredentials(Credentials): SCHEMA = BIGQUERY_CREDENTIALS_CONTRACT @@ -242,6 +264,9 @@ class BigQueryCredentials(Credentials): def type(self): return 'bigquery' + def _connection_keys(self): + return ('method', 'project', 'schema', 'location') + CREDENTIALS_MAPPING = { 'postgres': PostgresCredentials, From 3434ad9ca0624fe4192f2702fb94307525d05aaa Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 4 Dec 2018 14:34:52 -0700 Subject: [PATCH 02/12] Bypass project loading in init, we do it anyway --- dbt/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt/main.py b/dbt/main.py index 0434db9cae3..03435a340c8 100644 --- a/dbt/main.py +++ b/dbt/main.py @@ -162,8 +162,9 @@ def run_from_args(parsed): task = None cfg = None - if parsed.which == 'init': - # bypass looking for a project file if we're running `dbt init` + if parsed.which in ('init', 'debug'): + # bypass looking for a project file if we're running `dbt init` or + # `dbt debug` task = parsed.cls(args=parsed) else: nearest_project_dir = get_nearest_project_dir() From d4c2dfedb271679226249b15c1b22f148d91af65 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Tue, 4 Dec 2018 15:01:11 -0700 Subject: [PATCH 03/12] dbt debug --- dbt/config.py | 12 +-- dbt/task/debug.py | 268 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 250 insertions(+), 30 deletions(-) diff --git a/dbt/config.py b/dbt/config.py index 843467c2fc0..af6e471167f 100644 --- a/dbt/config.py +++ b/dbt/config.py @@ -543,7 +543,7 @@ def _credentials_from_profile(profile, profile_name, target_name): return credentials @staticmethod - def _pick_profile_name(args_profile_name, project_profile_name=None): + def pick_profile_name(args_profile_name, project_profile_name=None): profile_name = project_profile_name if args_profile_name is not None: profile_name = args_profile_name @@ -606,8 +606,8 @@ def from_credentials(cls, credentials, threads, profile_name, target_name, return profile @classmethod - def _render_profile(cls, raw_profile, profile_name, target_override, - cli_vars): + def render_profile(cls, raw_profile, profile_name, target_override, + cli_vars): """This is a containment zone for the hateful way we're rendering profiles. """ @@ -664,7 +664,7 @@ def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars, """ # user_cfg is not rendered since it only contains booleans. # TODO: should it be, and the values coerced to bool? - target_name, profile_data = cls._render_profile( + target_name, profile_data = cls.render_profile( raw_profile, profile_name, target_override, cli_vars ) @@ -749,8 +749,8 @@ def from_args(cls, args, project_profile_name=None, cli_vars=None): profiles_dir = getattr(args, 'profiles_dir', PROFILES_DIR) target_override = getattr(args, 'target', None) raw_profiles = read_profile(profiles_dir) - profile_name = cls._pick_profile_name(args.profile, - project_profile_name) + profile_name = cls.pick_profile_name(args.profile, + project_profile_name) return cls.from_raw_profiles( raw_profiles=raw_profiles, diff --git a/dbt/task/debug.py b/dbt/task/debug.py index a575791f6ff..dbaa2bc06e1 100644 --- a/dbt/task/debug.py +++ b/dbt/task/debug.py @@ -1,10 +1,18 @@ +# coding=utf-8 +import os +import platform import pprint +import sys from dbt.logger import GLOBAL_LOGGER as logger import dbt.clients.system import dbt.config import dbt.utils import dbt.exceptions +from dbt.adapters.factory import get_adapter +from dbt.version import get_installed_version +from dbt.config import Project, Profile +from dbt.clients.yaml_helper import load_yaml_text from dbt.task.base_task import BaseTask @@ -12,47 +20,259 @@ {open_cmd} {profiles_dir}""" +ONLY_PROFILE_MESSAGE = ''' +A `dbt_project.yml` file was not found in this directory. +Using the only profile `{}`. +'''.lstrip() + +MULTIPLE_PROFILE_MESSAGE = ''' +A `dbt_project.yml` file was not found in this directory. +dbt found the following profiles: +{} + +To debug one of these profiles, run: +dbt debug --profile [profile-name] +'''.lstrip() + +COULD_NOT_CONNECT_MESSAGE = ''' +dbt was unable to connect to the specified database. +The database returned the following error: + + >{} + +Check your database credentials and try again. For more information, visit: +https://docs.getdbt.com/docs/configure-your-profile +'''.lstrip() + + +MISSING_PROFILE_MESSAGE = ''' +dbt looked for a profiles.yml file in /Users/drew/.dbt/profiles.yml, but did +not find one. For more information on configuring your profile, consult the +documentation: + +https://docs.getdbt.com/docs/configure-your-profile +'''.lstrip() + +FILE_NOT_FOUND = 'file not found' + class DebugTask(BaseTask): + def __init__(self, args, config=None): + super(DebugTask, self).__init__(args, config) + self.profiles_dir = getattr(self.args, 'profiles_dir', + dbt.config.PROFILES_DIR) + self.profile_path = os.path.join(self.profiles_dir, 'profiles.yml') + self.project_path = os.path.join(os.getcwd(), 'dbt_project.yml') + self.cli_vars = dbt.utils.parse_cli_vars( + getattr(self.args, 'vars', '{}') + ) + + # set by _load_* + self.profile = None + self.profile_fail_details = '' + self.raw_profile_data = None + self.profile_name = None + self.project = None + self.project_fail_details = '' + self.messages = [] + + @property + def project_profile(self): + if self.project is None: + return None + return self.project.profile_name + def path_info(self): open_cmd = dbt.clients.system.open_dir_cmd() - profiles_dir = dbt.config.PROFILES_DIR message = PROFILE_DIR_MESSAGE.format( open_cmd=open_cmd, - profiles_dir=profiles_dir + profiles_dir=self.profiles_dir ) logger.info(message) - def diag(self): - # if we got here, a 'dbt_project.yml' does exist, but we have not tried - # to parse it. - project_profile = None - cli_vars = dbt.utils.parse_cli_vars(getattr(self.args, 'vars', '{}')) + def run(self): + version = get_installed_version().to_version_string(skip_matcher=True) + print('dbt version: {}'.format(version)) + print('python version: {}'.format(sys.version.split()[0])) + print('python path: {}'.format(sys.executable)) + print('os info: {}'.format(platform.platform())) + print('Using profiles.yml file at {}'.format(self.profile_path)) + print('') + self.test_configuration() + self.test_dependencies() + self.test_connection() + + for message in self.messages: + print(message) + print('') + + def _load_project(self): + if not os.path.exists(self.project_path): + self.project_fail_details = FILE_NOT_FOUND + return '✗ not found' try: - project = dbt.config.Project.from_current_directory(cli_vars) - project_profile = project.profile_name + self.project = Project.from_current_directory(self.cli_vars) except dbt.exceptions.DbtConfigError as exc: - project = 'ERROR loading project: {!s}'.format(exc) + self.project_fail_details = str(exc) + return '✗ invalid' + + return '✓ found and valid' + + def _profile_found(self): + if not self.raw_profile_data: + return '✗ not found' + if self.profile_name in self.raw_profile_data: + return '✓ found' + else: + return '✗ not found' + + def _target_found(self): + requirements = (self.raw_profile_data and self.profile_name and + self.target_name) + if not requirements: + return '✗ not found' + if self.profile_name not in self.raw_profile_data: + return '✗ not found' + profiles = self.raw_profile_data[self.profile_name]['outputs'] + if self.target_name not in profiles: + return '✗ not found' + return '✓ found' + + def _choose_profile_name(self): + assert self.project or self.project_fail_details, \ + '_load_project() required' + + project_profile = None + if self.project: + project_profile = self.project.profile_name + + args_profile = getattr(self.args, 'profile', None) - # log the profile we decided on as well, if it's available. try: - profile = dbt.config.Profile.from_args(self.args, project_profile, - cli_vars) - except dbt.exceptions.DbtConfigError as exc: - profile = 'ERROR loading profile: {!s}'.format(exc) + return Profile.pick_profile_name(args_profile, project_profile) + except dbt.exceptions.DbtConfigError: + pass + # try to guess - logger.info("args: {}".format(self.args)) - logger.info("") - logger.info("project:\n{!s}".format(project)) - logger.info("") - logger.info("profile:\n{!s}".format(profile)) + if self.raw_profile_data: + profiles = [k for k in self.raw_profile_data if k != 'config'] + if len(profiles) == 0: + self.messages.append('The profiles.yml has no profiles') + elif len(profiles) == 1: + self.messages.append(ONLY_PROFILE_MESSAGE.format(profiles[0])) + return profiles[0] + else: + self.messages.append(MULTIPLE_PROFILE_MESSAGE.format( + '\n'.join(' - {}'.format(o) for o in profiles) + )) + return None - def run(self): + def _choose_target_name(self): + has_raw_profile = (self.raw_profile_data and self.profile_name and + self.profile_name in self.raw_profile_data) + if has_raw_profile: + raw_profile = self.raw_profile_data[self.profile_name] - if self.args.config_dir: - self.path_info() + target_name, _ = Profile.render_profile( + raw_profile, self.profile_name, + getattr(self.args, 'target', None), self.cli_vars + ) + return target_name + return None + + def _load_profile(self): + if not os.path.exists(self.profile_path): + self.profile_fail_details = FILE_NOT_FOUND + self.messages.append(MISSING_PROFILE_MESSAGE) + return '✗ not found' + + try: + raw_profile_data = load_yaml_text( + dbt.clients.system.load_file_contents(self.profile_path) + ) + except Exception: + pass # we'll report this when we try to load the profile for real else: - self.diag() + if isinstance(raw_profile_data, dict): + self.raw_profile_data = raw_profile_data + + self.profile_name = self._choose_profile_name() + self.target_name = self._choose_target_name() + try: + self.profile = Profile.from_args(self.args, self.profile_name, + self.cli_vars) + except dbt.exceptions.DbtConfigError as exc: + self.profile_fail_details = str(exc) + return '✗ invalid' + + return '✓ found and valid' + + def test_git(self): + try: + dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help']) + except dbt.exceptions.ExecutableError as exc: + self.messages.append('Error from git --help: {!s}'.format(exc)) + return '✗ error' + return '✓ found' + + def test_dependencies(self): + print('Required dependencies:') + print(' - git [{}]'.format(self.test_git())) + print('') + + def test_configuration(self): + project_status = self._load_project() + profile_status = self._load_profile() + print('Configuration:') + print(' profiles.yml file [{}]'.format(profile_status)) + print(' dbt_project.yml file [{}]'.format(project_status)) + # skip profile stuff if we can't find a profile name + if self.profile_name is not None: + print(' profile: {} [{}]'.format(self.profile_name, + self._profile_found())) + print(' target: {} [{}]'.format(self.target_name, + self._target_found())) + print('') + self._log_project_fail() + self._log_profile_fail() + + def _log_project_fail(self): + if not self.project_fail_details: + return + if self.project_fail_details == FILE_NOT_FOUND: + return + print('Project loading failed for the following reason:') + print(self.project_fail_details) + print('') + + def _log_profile_fail(self): + if not self.profile_fail_details: + return + if self.profile_fail_details == FILE_NOT_FOUND: + return + if self.profile_name is None: + return # we expect an error (no profile provided) + print('Profile loading failed for the following reason:') + print(self.profile_fail_details) + print('') + + def _connection_result(self): + adapter = get_adapter(self.profile) + try: + adapter.execute('select 1 as id') + except Exception as exc: + self.messages.append(COULD_NOT_CONNECT_MESSAGE.format(str(exc))) + return '✗ error' + return '✓ connection ok' + + def test_connection(self): + if not self.profile: + return + print('Connection:') + for k, v in self.profile.credentials.connection_info(): + print(' {}: {}'.format(k, v)) + print(' Connection test: {}'.format(self._connection_result())) + print('') From 6935a4a2e4eeda654630f822a036f1bb7e263a68 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 5 Dec 2018 08:16:41 -0700 Subject: [PATCH 04/12] pr feedback: add pretty colors --- dbt/task/debug.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/dbt/task/debug.py b/dbt/task/debug.py index dbaa2bc06e1..9ad4453bdff 100644 --- a/dbt/task/debug.py +++ b/dbt/task/debug.py @@ -13,6 +13,7 @@ from dbt.version import get_installed_version from dbt.config import Project, Profile from dbt.clients.yaml_helper import load_yaml_text +from dbt.ui.printer import green, red from dbt.task.base_task import BaseTask @@ -111,35 +112,35 @@ def run(self): def _load_project(self): if not os.path.exists(self.project_path): self.project_fail_details = FILE_NOT_FOUND - return '✗ not found' + return red('✗ not found') try: self.project = Project.from_current_directory(self.cli_vars) except dbt.exceptions.DbtConfigError as exc: self.project_fail_details = str(exc) - return '✗ invalid' + return red('✗ invalid') - return '✓ found and valid' + return green('✓ found and valid') def _profile_found(self): if not self.raw_profile_data: - return '✗ not found' + return red('✗ not found') if self.profile_name in self.raw_profile_data: - return '✓ found' + return green('✓ found') else: - return '✗ not found' + return red('✗ not found') def _target_found(self): requirements = (self.raw_profile_data and self.profile_name and self.target_name) if not requirements: - return '✗ not found' + return red('✗ not found') if self.profile_name not in self.raw_profile_data: - return '✗ not found' + return red('✗ not found') profiles = self.raw_profile_data[self.profile_name]['outputs'] if self.target_name not in profiles: - return '✗ not found' - return '✓ found' + return red('✗ not found') + return green('✓ found') def _choose_profile_name(self): assert self.project or self.project_fail_details, \ @@ -187,7 +188,7 @@ def _load_profile(self): if not os.path.exists(self.profile_path): self.profile_fail_details = FILE_NOT_FOUND self.messages.append(MISSING_PROFILE_MESSAGE) - return '✗ not found' + return red('✗ not found') try: raw_profile_data = load_yaml_text( @@ -206,17 +207,17 @@ def _load_profile(self): self.cli_vars) except dbt.exceptions.DbtConfigError as exc: self.profile_fail_details = str(exc) - return '✗ invalid' + return red('✗ invalid') - return '✓ found and valid' + return green('✓ found and valid') def test_git(self): try: dbt.clients.system.run_cmd(os.getcwd(), ['git', '--help']) except dbt.exceptions.ExecutableError as exc: self.messages.append('Error from git --help: {!s}'.format(exc)) - return '✗ error' - return '✓ found' + return red('✗ error') + return green('✓ found') def test_dependencies(self): print('Required dependencies:') @@ -265,8 +266,8 @@ def _connection_result(self): adapter.execute('select 1 as id') except Exception as exc: self.messages.append(COULD_NOT_CONNECT_MESSAGE.format(str(exc))) - return '✗ error' - return '✓ connection ok' + return red('✗ error') + return green('✓ connection ok') def test_connection(self): if not self.profile: From 9938af158015bd57c1e9c7a1fc412e73e731bf68 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 5 Dec 2018 08:33:16 -0700 Subject: [PATCH 05/12] pr feedback --- dbt/links.py | 2 +- dbt/task/debug.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dbt/links.py b/dbt/links.py index 3be2c275d9e..84def357631 100644 --- a/dbt/links.py +++ b/dbt/links.py @@ -1,2 +1,2 @@ - +ProfileConfigDocs = 'https://docs.getdbt.com/docs/configure-your-profile' SnowflakeQuotingDocs = 'https://docs.getdbt.com/v0.10/docs/configuring-quoting' diff --git a/dbt/task/debug.py b/dbt/task/debug.py index 9ad4453bdff..01f395ee0f6 100644 --- a/dbt/task/debug.py +++ b/dbt/task/debug.py @@ -9,6 +9,7 @@ import dbt.config import dbt.utils import dbt.exceptions +from dbt.links import ProfileConfigDocs from dbt.adapters.factory import get_adapter from dbt.version import get_installed_version from dbt.config import Project, Profile @@ -39,19 +40,19 @@ dbt was unable to connect to the specified database. The database returned the following error: - >{} + >{err} Check your database credentials and try again. For more information, visit: -https://docs.getdbt.com/docs/configure-your-profile +{url} '''.lstrip() MISSING_PROFILE_MESSAGE = ''' -dbt looked for a profiles.yml file in /Users/drew/.dbt/profiles.yml, but did +dbt looked for a profiles.yml file in {path}, but did not find one. For more information on configuring your profile, consult the documentation: -https://docs.getdbt.com/docs/configure-your-profile +{url} '''.lstrip() FILE_NOT_FOUND = 'file not found' @@ -187,7 +188,9 @@ def _choose_target_name(self): def _load_profile(self): if not os.path.exists(self.profile_path): self.profile_fail_details = FILE_NOT_FOUND - self.messages.append(MISSING_PROFILE_MESSAGE) + self.messages.append(MISSING_PROFILE_MESSAGE.format( + path=self.profile_path, url=ProfileConfigDocs + )) return red('✗ not found') try: @@ -265,7 +268,10 @@ def _connection_result(self): try: adapter.execute('select 1 as id') except Exception as exc: - self.messages.append(COULD_NOT_CONNECT_MESSAGE.format(str(exc))) + self.messages.append(COULD_NOT_CONNECT_MESSAGE.format( + err=str(exc), + url=ProfileConfigDocs + )) return red('✗ error') return green('✓ connection ok') From 80232ff8e8e58ab4a061618bbcc8958f2aa2395a Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 14 Nov 2018 23:06:05 -0500 Subject: [PATCH 06/12] (fixes #744) deprecate sql_where and provide an alternative --- dbt/compilation.py | 7 ++++ dbt/deprecations.py | 13 +++++- .../macros/etc/is_incremental.sql | 10 +++++ .../incremental/bigquery_incremental.sql | 4 +- .../incremental/incremental.sql | 5 ++- dbt/links.py | 1 + .../models/incremental.sql | 7 +++- .../models/incremental_deprecated.sql | 8 ++++ .../001_simple_copy_test/test_simple_copy.py | 40 ++++++++++--------- 9 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 dbt/include/global_project/macros/etc/is_incremental.sql create mode 100644 test/integration/001_simple_copy_test/models/incremental_deprecated.sql diff --git a/dbt/compilation.py b/dbt/compilation.py index f339c5f9152..9563a381cc9 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -232,6 +232,12 @@ def _check_resource_uniqueness(cls, manifest): names_resources[name] = node alias_resources[alias] = node + def warn_for_deprecated_configs(self, manifest): + for unique_id, node in manifest.nodes.items(): + is_model = node.resource_type == NodeType.Model + if is_model and 'sql_where' in node.config: + dbt.deprecations.warn('sql_where') + def compile(self): linker = Linker() @@ -247,6 +253,7 @@ def compile(self): disabled_fqns = [n.fqn for n in manifest.disabled] self.config.warn_for_unused_resource_config_paths(resource_fqns, disabled_fqns) + self.warn_for_deprecated_configs(manifest) self.link_graph(linker, manifest) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index 00d656b7be0..eeec80473f5 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -1,4 +1,5 @@ from dbt.logger import GLOBAL_LOGGER as logger +import dbt.links class DBTDeprecation(object): @@ -25,6 +26,15 @@ class DBTRepositoriesDeprecation(DBTDeprecation): {recommendation} """ +class SqlWhereDeprecation(DBTDeprecation): + name = "sql_where" + description = """\ +The `sql_where` option for incremental models is deprecated and will be + removed in a future release. Check the docs for more information + + {} + """.format(dbt.links.IncrementalDocs) + class SeedDropExistingDeprecation(DBTDeprecation): name = 'drop-existing' @@ -50,7 +60,8 @@ def warn(name, *args, **kwargs): deprecations_list = [ DBTRepositoriesDeprecation(), - SeedDropExistingDeprecation() + SeedDropExistingDeprecation(), + SqlWhereDeprecation(), ] deprecations = {d.name: d for d in deprecations_list} diff --git a/dbt/include/global_project/macros/etc/is_incremental.sql b/dbt/include/global_project/macros/etc/is_incremental.sql new file mode 100644 index 00000000000..fb761cd7c53 --- /dev/null +++ b/dbt/include/global_project/macros/etc/is_incremental.sql @@ -0,0 +1,10 @@ + +{% macro is_incremental() %} + {#-- do not run introspective queries in parsing #} + {% if not execute %} + {{ return(False) }} + {% else %} + {% set relation = adapter.get_relation(this.schema, this.table) %} + {{ return(relation is not none and not flags.FULL_REFRESH) }} + {% endif %} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/incremental/bigquery_incremental.sql b/dbt/include/global_project/macros/materializations/incremental/bigquery_incremental.sql index e24a9caa132..d43b5f7a645 100644 --- a/dbt/include/global_project/macros/materializations/incremental/bigquery_incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/bigquery_incremental.sql @@ -37,7 +37,9 @@ select * from ( {{ sql }} ) - where ({{ sql_where }}) or ({{ sql_where }}) is null + {% if sql_where %} + where ({{ sql_where }}) or ({{ sql_where }}) is null + {% endif %} ) {%- endset -%} diff --git a/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/incremental/incremental.sql index 8df0264a8a4..ec64645ab9e 100644 --- a/dbt/include/global_project/macros/materializations/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/incremental/incremental.sql @@ -12,7 +12,7 @@ {%- endmacro %} {% materialization incremental, default -%} - {%- set sql_where = config.require('sql_where') -%} + {%- set sql_where = config.get('sql_where') -%} {%- set unique_key = config.get('unique_key') -%} {%- set identifier = model['alias'] -%} @@ -61,8 +61,11 @@ select * from ( {{ sql }} ) as dbt_incr_sbq + + {% if sql_where %} where ({{ sql_where }}) or ({{ sql_where }}) is null + {% endif %} {%- endset %} {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} diff --git a/dbt/links.py b/dbt/links.py index 3be2c275d9e..39fb0d4e0d7 100644 --- a/dbt/links.py +++ b/dbt/links.py @@ -1,2 +1,3 @@ SnowflakeQuotingDocs = 'https://docs.getdbt.com/v0.10/docs/configuring-quoting' +IncrementalDocs = 'https://docs.getdbt.com/docs/configuring-incremental-models' diff --git a/test/integration/001_simple_copy_test/models/incremental.sql b/test/integration/001_simple_copy_test/models/incremental.sql index 72e0c872e6b..020bf35163e 100644 --- a/test/integration/001_simple_copy_test/models/incremental.sql +++ b/test/integration/001_simple_copy_test/models/incremental.sql @@ -1,8 +1,11 @@ {{ config( - materialized = "incremental", - sql_where = "id>(select max(id) from {{this}})" + materialized = "incremental" ) }} select * from {{ ref('seed') }} + +{% if is_incremental() %} + where id > (select max(id) from {{this}}) +{% endif %} diff --git a/test/integration/001_simple_copy_test/models/incremental_deprecated.sql b/test/integration/001_simple_copy_test/models/incremental_deprecated.sql new file mode 100644 index 00000000000..72e0c872e6b --- /dev/null +++ b/test/integration/001_simple_copy_test/models/incremental_deprecated.sql @@ -0,0 +1,8 @@ +{{ + config( + materialized = "incremental", + sql_where = "id>(select max(id) from {{this}})" + ) +}} + +select * from {{ ref('seed') }} diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index 7cc4ed580e4..ea6194a196e 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -25,17 +25,17 @@ def test__postgres__simple_copy(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) - self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized"]) + self.assertManyTablesEqual(["seed", "view_model", "incremental", "incremental_deprecated", "materialized"]) self.use_default_project({"data-paths": [self.dir("seed-update")]}) results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) - self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized"]) + self.assertManyTablesEqual(["seed", "view_model", "incremental", "incremental_deprecated", "materialized"]) @use_profile("postgres") def test__postgres__dbt_doesnt_run_empty_models(self): @@ -44,7 +44,7 @@ def test__postgres__dbt_doesnt_run_empty_models(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) models = self.get_models_in_schema() @@ -58,13 +58,13 @@ def test__snowflake__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED"]) + self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "INCREMENTAL_DEPRECATED", "MATERIALIZED"]) self.use_default_project({"data-paths": [self.dir("seed-update")]}) self.run_dbt(["seed"]) self.run_dbt() - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED"]) + self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "INCREMENTAL_DEPRECATED", "MATERIALIZED"]) @use_profile("snowflake") def test__snowflake__simple_copy__quoting_off(self): @@ -76,9 +76,9 @@ def test__snowflake__simple_copy__quoting_off(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED"]) + self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "INCREMENTAL_DEPRECATED", "MATERIALIZED"]) self.use_default_project({ "data-paths": [self.dir("seed-update")], @@ -87,9 +87,9 @@ def test__snowflake__simple_copy__quoting_off(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED"]) + self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "INCREMENTAL_DEPRECATED", "MATERIALIZED"]) @use_profile("snowflake") def test__snowflake__seed__quoting_switch(self): @@ -114,9 +114,10 @@ def test__bigquery__simple_copy(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","incremental_deprecated") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -125,9 +126,10 @@ def test__bigquery__simple_copy(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) self.assertTablesEqual("seed","view_model") + self.assertTablesEqual("seed","incremental_deprecated") self.assertTablesEqual("seed","incremental") self.assertTablesEqual("seed","materialized") @@ -150,9 +152,9 @@ def test__snowflake__simple_copy__quoting_on(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) - self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized"]) + self.assertManyTablesEqual(["seed", "view_model", "incremental", "incremental_deprecated", "materialized"]) self.use_default_project({ "data-paths": [self.dir("seed-update")], @@ -160,9 +162,9 @@ def test__snowflake__simple_copy__quoting_on(self): results = self.run_dbt(["seed"]) self.assertEqual(len(results), 1) results = self.run_dbt() - self.assertEqual(len(results), 6) + self.assertEqual(len(results), 7) - self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized"]) + self.assertManyTablesEqual(["seed", "view_model", "incremental", "incremental_deprecated", "materialized"]) class BaseLowercasedSchemaTest(BaseTestSimpleCopy): @@ -180,13 +182,13 @@ def test__snowflake__simple_copy(self): self.run_dbt(["seed"]) self.run_dbt() - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED"]) + self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "INCREMENTAL_DEPRECATED", "MATERIALIZED"]) self.use_default_project({"data-paths": [self.dir("seed-update")]}) self.run_dbt(["seed"]) self.run_dbt() - self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED"]) + self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "INCREMENTAL_DEPRECATED", "MATERIALIZED"]) class TestSnowflakeSimpleLowercasedSchemaQuoted(BaseLowercasedSchemaTest): From 009eaa3a59a63ddd8808ef378ffc8170c17c1f69 Mon Sep 17 00:00:00 2001 From: Drew Banin Date: Wed, 5 Dec 2018 11:06:04 -0500 Subject: [PATCH 07/12] pep8 --- dbt/deprecations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/deprecations.py b/dbt/deprecations.py index eeec80473f5..71f9db25495 100644 --- a/dbt/deprecations.py +++ b/dbt/deprecations.py @@ -26,6 +26,7 @@ class DBTRepositoriesDeprecation(DBTDeprecation): {recommendation} """ + class SqlWhereDeprecation(DBTDeprecation): name = "sql_where" description = """\ From 0f1520c3922a8329ccbdefc237d5e7a5d13c3a37 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 5 Dec 2018 09:23:11 -0700 Subject: [PATCH 08/12] re-add call to path_info --- dbt/task/debug.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt/task/debug.py b/dbt/task/debug.py index 01f395ee0f6..7660c4aa47b 100644 --- a/dbt/task/debug.py +++ b/dbt/task/debug.py @@ -95,6 +95,10 @@ def path_info(self): logger.info(message) def run(self): + if self.args.config_dir: + self.path_info() + return + version = get_installed_version().to_version_string(skip_matcher=True) print('dbt version: {}'.format(version)) print('python version: {}'.format(sys.version.split()[0])) From eb504ae866c35d3bdc7f3913c7eb03b4e42f0114 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 5 Dec 2018 09:20:44 -0700 Subject: [PATCH 09/12] use pypi for the latest version instead of git --- dbt/version.py | 48 ++++++++++++------------------ test/unit/test_version.py | 62 ++++++++------------------------------- 2 files changed, 30 insertions(+), 80 deletions(-) diff --git a/dbt/version.py b/dbt/version.py index 2cc558dd5ae..2988f61ff72 100644 --- a/dbt/version.py +++ b/dbt/version.py @@ -1,43 +1,31 @@ +import json import re -import dbt.semver - -try: - # For Python 3.0 and later - from urllib.request import urlopen -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen +import requests -REMOTE_VERSION_FILE = \ - 'https://raw.githubusercontent.com/fishtown-analytics/dbt/' \ - 'master/.bumpversion.cfg' +import dbt.exceptions +import dbt.semver -def get_version_string_from_text(contents): - matches = re.search(r"current_version = ([\.0-9a-z]+)", contents) - if matches is None or len(matches.groups()) != 1: - return "" - version = matches.groups()[0] - return version +PYPI_VERSION_URL = 'https://pypi.org/pypi/dbt/json' -def get_remote_version_file_contents(url=REMOTE_VERSION_FILE): +def get_latest_version(): + resp = requests.get(PYPI_VERSION_URL) try: - f = urlopen(url) - contents = f.read() - except Exception: - contents = '' - if hasattr(contents, 'decode'): - contents = contents.decode('utf-8') - return contents + data = resp.json() + except json.JSONDecodeError: + raise dbt.exceptions.RuntimeError( + 'Got invalid data from pypi while querying versions: not json' + ) + try: + version_string = data['info']['version'] + except KeyError: + raise dbt.exceptions.RuntimeError( + 'Got invalid data from pypi while querying versions: bad data' + ) -def get_latest_version(): - contents = get_remote_version_file_contents() - if contents == '': - return None - version_string = get_version_string_from_text(contents) return dbt.semver.VersionSpecifier.from_version_string(version_string) diff --git a/test/unit/test_version.py b/test/unit/test_version.py index 4b63844edaf..f392646b79b 100644 --- a/test/unit/test_version.py +++ b/test/unit/test_version.py @@ -9,19 +9,9 @@ class VersionTest(unittest.TestCase): @patch("dbt.version.__version__", "0.10.0") - def test_versions_equal(self): - - dbt.version.get_remote_version_file_contents = MagicMock( - return_value=""" - [bumpversion] - current_version = 0.10.0 - commit = True - tag = True - - [bumpversion:file:setup.py] - - [bumpversion:file:dbt/version.py] - """) + @patch('dbt.version.requests.get') + def test_versions_equal(self, mock_get): + mock_get.return_value.json.return_value = {'info': {'version': '0.10.0'}} latest_version = dbt.version.get_latest_version() installed_version = dbt.version.get_installed_version() @@ -37,18 +27,9 @@ def test_versions_equal(self): expected_version_information) @patch("dbt.version.__version__", "0.10.2-a1") - def test_installed_version_greater(self): - dbt.version.get_remote_version_file_contents = MagicMock( - return_value=""" - [bumpversion] - current_version = 0.10.1 - commit = True - tag = True - - [bumpversion:file:setup.py] - - [bumpversion:file:dbt/version.py] - """) + @patch('dbt.version.requests.get') + def test_installed_version_greater(self, mock_get): + mock_get.return_value.json.return_value = {'info': {'version': '0.10.1'}} latest_version = dbt.version.get_latest_version() installed_version = dbt.version.get_installed_version() @@ -63,18 +44,9 @@ def test_installed_version_greater(self): expected_version_information) @patch("dbt.version.__version__", "0.9.5") - def test_installed_version_lower(self): - dbt.version.get_remote_version_file_contents = MagicMock( - return_value=""" - [bumpversion] - current_version = 0.10.0 - commit = True - tag = True - - [bumpversion:file:setup.py] - - [bumpversion:file:dbt/version.py] - """) + @patch('dbt.version.requests.get') + def test_installed_version_lower(self, mock_get): + mock_get.return_value.json.return_value = {'info': {'version': '0.10.0'}} latest_version = dbt.version.get_latest_version() installed_version = dbt.version.get_installed_version() @@ -92,20 +64,10 @@ def test_installed_version_lower(self): # suppress having version info printed to the screen during tests. @patch('sys.stderr') - def test_dbt_version_flag(self, stderr): - dbt.version.get_remote_version_file_contents = MagicMock( - return_value=""" - [bumpversion] - current_version = 0.10.1 - commit = True - tag = True - - [bumpversion:file:setup.py] - - [bumpversion:file:dbt/version.py] - """) + @patch('dbt.version.requests.get') + def test_dbt_version_flag(self, mock_get, stderr): + mock_get.return_value.json.return_value = {'info': {'version': '0.10.1'}} with self.assertRaises(SystemExit) as exc: dbt.main.handle_and_check(['--version']) self.assertEqual(exc.exception.code, 0) - From 8ea9c68be0cfbd330805460a1e83d35214da3c1e Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 5 Dec 2018 10:00:30 -0700 Subject: [PATCH 10/12] PR feedback --- dbt/version.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/dbt/version.py b/dbt/version.py index 2988f61ff72..48da2832748 100644 --- a/dbt/version.py +++ b/dbt/version.py @@ -14,17 +14,9 @@ def get_latest_version(): resp = requests.get(PYPI_VERSION_URL) try: data = resp.json() - except json.JSONDecodeError: - raise dbt.exceptions.RuntimeError( - 'Got invalid data from pypi while querying versions: not json' - ) - - try: version_string = data['info']['version'] - except KeyError: - raise dbt.exceptions.RuntimeError( - 'Got invalid data from pypi while querying versions: bad data' - ) + except (json.JSONDecodeError, KeyError): + return None return dbt.semver.VersionSpecifier.from_version_string(version_string) @@ -49,7 +41,7 @@ def get_version_information(): if latest is None: return ("{}The latest version of dbt could not be determined!\n" "Make sure that the following URL is accessible:\n{}" - .format(version_msg, REMOTE_VERSION_FILE)) + .format(version_msg, PYPI_VERSION_URL)) if installed == latest: return "{}Up to date!".format(version_msg) From 9b8e8ff17a8525c412bb2c5988d515ef72d69e52 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Wed, 5 Dec 2018 10:12:57 -0700 Subject: [PATCH 11/12] more pr feedback --- dbt/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/version.py b/dbt/version.py index 48da2832748..319f36d88ff 100644 --- a/dbt/version.py +++ b/dbt/version.py @@ -11,11 +11,11 @@ def get_latest_version(): - resp = requests.get(PYPI_VERSION_URL) try: + resp = requests.get(PYPI_VERSION_URL) data = resp.json() version_string = data['info']['version'] - except (json.JSONDecodeError, KeyError): + except (json.JSONDecodeError, KeyError, requests.RequestException): return None return dbt.semver.VersionSpecifier.from_version_string(version_string) From 65729c4acc5d5c87dfb8feb0ec65d52e4f5400f1 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Wed, 5 Dec 2018 12:15:50 -0500 Subject: [PATCH 12/12] =?UTF-8?q?Bump=20version:=200.12.1=20=E2=86=92=200.?= =?UTF-8?q?12.2a1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- dbt/version.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 32178aaf456..62b0bf0a062 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.12.1 +current_version = 0.12.2a1 parse = (?P\d+) \.(?P\d+) \.(?P\d+) diff --git a/dbt/version.py b/dbt/version.py index 2cc558dd5ae..eaf5070468c 100644 --- a/dbt/version.py +++ b/dbt/version.py @@ -77,5 +77,5 @@ def get_version_information(): .format(version_msg)) -__version__ = '0.12.1' +__version__ = '0.12.2a1' installed = get_installed_version() diff --git a/setup.py b/setup.py index cb9091e4506..ab763c0965f 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() package_name = "dbt" -package_version = "0.12.1" +package_version = "0.12.2a1" description = """dbt (data build tool) is a command line tool that helps \ analysts and engineers transform data in their warehouse more effectively"""