Merge branch 'dev/stephen-girard' into fix/incremental-from-view

core/dbt/adapters/base/connections.py

@@ -296,7 +296,7 @@ def _rollback(self, connection):
         operation does not require the lock.
         """
         if dbt.flags.STRICT_MODE:
-            Connection(**connection)
+            assert isinstance(connection, Connection)
 
         if connection.transaction_open is False:
             raise dbt.exceptions.InternalException(

core/dbt/adapters/sql/connections.py

@@ -3,6 +3,7 @@
 
 import dbt.clients.agate_helper
 import dbt.exceptions
+from dbt.contracts.connection import Connection
 from dbt.adapters.base import BaseConnectionManager
 from dbt.compat import abstractclassmethod
 from dbt.logger import GLOBAL_LOGGER as logger

core/dbt/compilation.py

@@ -71,8 +71,8 @@ def recursively_prepend_ctes(model, manifest):
         return (model, model.extra_ctes, manifest)
 
     if dbt.flags.STRICT_MODE:
-        # ensure that all the nodes in this manifest are compiled
-        CompiledGraph(**manifest.to_flat_graph())
+        # ensure that the cte we're adding to is compiled
+        CompiledNode(**model.serialize())
 
     prepended_ctes = []
 
@@ -81,7 +81,6 @@ def recursively_prepend_ctes(model, manifest):
         cte_to_add = manifest.nodes.get(cte_id)
         cte_to_add, new_prepended_ctes, manifest = recursively_prepend_ctes(
             cte_to_add, manifest)
-
         _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
         new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.get('name'))
         sql = ' {} as (\n{}\n)'.format(new_cte_name, cte_to_add.compiled_sql)

core/dbt/config/project.py

@@ -13,6 +13,7 @@
 from dbt.exceptions import RecursionException
 from dbt.exceptions import SemverException
 from dbt.exceptions import ValidationException
+from dbt.exceptions import warn_or_error
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.semver import VersionSpecifier
 from dbt.semver import versions_compatible
@@ -417,7 +418,7 @@ def warn_for_unused_resource_config_paths(self, resource_fqns, disabled):
             len(unused),
             '\n'.join('- {}'.format('.'.join(u)) for u in unused)
         )
-        logger.info(printer.yellow(msg))
+        warn_or_error(msg, log_fmt=printer.yellow('{}'))
 
     def validate_version(self):
         """Ensure this package works with the installed version of dbt."""

core/dbt/deprecations.py

@@ -1,5 +1,6 @@
 from dbt.logger import GLOBAL_LOGGER as logger
 import dbt.links
+import dbt.flags
 
 
 class DBTDeprecation(object):
@@ -9,7 +10,9 @@ class DBTDeprecation(object):
     def show(self, *args, **kwargs):
         if self.name not in active_deprecations:
             desc = self.description.format(**kwargs)
-            logger.info("* Deprecation Warning: {}\n".format(desc))
+            dbt.exceptions.warn_or_error(
+                "* Deprecation Warning: {}\n".format(desc)
+            )
             active_deprecations.add(self.name)
 
 

core/dbt/exceptions.py

@@ -1,5 +1,6 @@
 from dbt.compat import basestring, builtins
 from dbt.logger import GLOBAL_LOGGER as logger
+import dbt.flags
 import re
 
 
@@ -580,6 +581,15 @@ def raise_not_implemented(msg):
     raise NotImplementedException(msg)
 
 
+def warn_or_error(msg, node=None, log_fmt=None):
+    if dbt.flags.WARN_ERROR:
+        raise_compiler_error(msg, node)
+    else:
+        if log_fmt is not None:
+            msg = log_fmt.format(msg)
+        logger.warning(msg)
+
+
 # Update this when a new function should be added to the
 # dbt context's `exceptions` key!
 CONTEXT_EXPORTS = {

core/dbt/flags.py

@@ -2,12 +2,14 @@
 NON_DESTRUCTIVE = False
 FULL_REFRESH = False
 USE_CACHE = True
+WARN_ERROR = False
 
 
 def reset():
-    global STRICT_MODE, NON_DESTRUCTIVE, FULL_REFRESH
+    global STRICT_MODE, NON_DESTRUCTIVE, FULL_REFRESH, USE_CACHE, WARN_ERROR
 
     STRICT_MODE = False
     NON_DESTRUCTIVE = False
     FULL_REFRESH = False
     USE_CACHE = True
+    WARN_ERROR = False

core/dbt/graph/selector.py

@@ -125,10 +125,11 @@ def warn_if_useless_spec(spec, nodes):
     if len(nodes) > 0:
         return
 
-    logger.info(
-        "* Spec='{}' does not identify any models and was ignored\n"
+    msg = (
+        "* Spec='{}' does not identify any models"
         .format(spec['raw'])
     )
+    dbt.exceptions.warn_or_error(msg, log_fmt='{} and was ignored\n')
 
 
 class NodeSelector(object):

core/dbt/main.py

@@ -284,6 +284,11 @@ def invoke_dbt(parsed):
 
     arg_drop_existing = getattr(parsed, 'drop_existing', False)
     arg_full_refresh = getattr(parsed, 'full_refresh', False)
+    flags.STRICT_MODE = getattr(parsed, 'strict', False)
+    flags.WARN_ERROR = (
+        flags.STRICT_MODE or
+        getattr(parsed, 'warn_error', False)
+    )
 
     if arg_drop_existing:
         dbt.deprecations.warn('drop-existing')
@@ -338,7 +343,16 @@ def parse_args(args):
         '--strict',
         action='store_true',
         help='''Run schema validations at runtime. This will surface
-        bugs in dbt, but may incur a performance penalty.''')
+        bugs in dbt, but may incur a performance penalty. This flag implies
+        --warn-error''')
+
+    p.add_argument(
+        '--warn-error',
+        action='store_true',
+        help='''If dbt would normally warn, instead raise an exception.
+        Examples include --models that selects nothing, deprecations,
+        configurations with no associated models, invalid test configurations,
+        and missing sources/refs in tests''')
 
     # if set, run dbt in single-threaded mode: thread count is ignored, and
     # calls go through `map` instead of the thread pool. This is useful for

core/dbt/parser/schemas.py

@@ -164,9 +164,8 @@ def warn_invalid(filepath, key, value, explain):
     msg = (
         "Invalid test config given in {} @ {}: {} {}"
     ).format(filepath, key, value, explain)
-    if dbt.flags.STRICT_MODE:
-        dbt.exceptions.raise_compiler_error(msg, value)
-    dbt.utils.compiler_warning(value, msg)
+    dbt.exceptions.warn_or_error(msg, value,
+                                 log_fmt='Compilation warning: {}\n')
 
 
 def _filter_validate(filepath, location, values, validate):

core/dbt/utils.py

@@ -11,7 +11,6 @@
 import os
 
 import dbt.exceptions
-import dbt.flags
 
 from dbt.include.global_project import PACKAGES
 from dbt.compat import basestring, DECIMALS
@@ -416,7 +415,7 @@ def invalid_ref_fail_unless_test(node, target_model_name,
         if disabled:
             logger.debug(msg)
         else:
-            logger.warning(msg)
+            dbt.exceptions.warn_or_error(msg)
 
     else:
         dbt.exceptions.ref_target_not_found(
@@ -429,7 +428,7 @@ def invalid_source_fail_unless_test(node, target_name, target_table_name):
     if node.get('resource_type') == NodeType.Test:
         msg = dbt.exceptions.source_disabled_message(node, target_name,
                                                      target_table_name)
-        logger.warning('WARNING: {}'.format(msg))
+        dbt.exceptions.warn_or_error(msg, log_fmt='WARNING: {}')
     else:
         dbt.exceptions.source_target_not_found(node, target_name,
                                                target_table_name)

plugins/bigquery/dbt/adapters/bigquery/impl.py

@@ -333,7 +333,7 @@ def execute_model(self, model, materialization, sql_override=None,
 
         if flags.STRICT_MODE:
             connection = self.connections.get(model.get('name'))
-            Connection(**connection)
+            assert isinstance(connection, Connection)
 
         if materialization == 'view':
             res = self._materialize_as_view(model)

test/integration/001_simple_copy_test/models/advanced_incremental.sql

@@ -1,7 +1,6 @@
 {{
   config(
     materialized = "incremental",
-    sql_where = "TRUE",
     unique_key = "id"
   )
 }}
@@ -10,7 +9,7 @@
 select *
 from {{ ref('seed') }}
 
-{% if adapter.already_exists(this.schema, this.table) and not flags.FULL_REFRESH %}
+{% if is_incremental() %}
 
     where id > (select max(id) from {{this}})