diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index ad41aa2047d..c8999452d9f 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -274,10 +274,6 @@ def finalize_and_validate(self: T) -> T: @dataclass class SourceConfig(BaseConfig): enabled: bool = True - quoting: Dict[str, Any] = field( - default_factory=dict, - metadata=MergeBehavior.Update.meta(), - ) @dataclass diff --git a/core/dbt/deprecations.py b/core/dbt/deprecations.py index 20df3919de7..baeffd79b53 100644 --- a/core/dbt/deprecations.py +++ b/core/dbt/deprecations.py @@ -92,23 +92,16 @@ class ModelsKeyNonModelDeprecation(DBTDeprecation): ''' -class BigQueryPartitionByStringDeprecation(DBTDeprecation): - _name = 'bq-partition-by-string' - - _description = ''' - As of dbt v0.16.0, the `partition_by` config in BigQuery accepts a - dictionary containing `field` and `data_type`. - - - - Provided partition_by: {raw_partition_by} - - - - dbt inferred: {inferred_partition_by} - +class DbtProjectYamlDeprecation(DBTDeprecation): + _name = 'dbt-project-yaml-v1' + _description = '''\ + The existing dbt_project.yml format has been deprecated. dbt_project.yml + has been upgraded to config version 2. A future version of dbt will remove + support for the existing ("version 1") format. + Documentation for dbt_project.yml version 2 can be found here: - For more information, see: - https://docs.getdbt.com/docs/upgrading-to-0-16-0 + DOCS LINK GOES HERE ''' @@ -154,7 +147,7 @@ def warn(name, *args, **kwargs): NotADictionaryDeprecation(), ColumnQuotingDeprecation(), ModelsKeyNonModelDeprecation(), - BigQueryPartitionByStringDeprecation(), + DbtProjectYamlDeprecation(), ] deprecations: Dict[str, DBTDeprecation] = { diff --git a/core/dbt/include/global_project/dbt_project.yml b/core/dbt/include/global_project/dbt_project.yml index 87bfe0dd8eb..dec6d7d452f 100644 --- a/core/dbt/include/global_project/dbt_project.yml +++ b/core/dbt/include/global_project/dbt_project.yml @@ -1,4 +1,4 @@ - +config-version: 2 name: dbt version: 1.0 diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index ecfe2f3b7d3..490ef47f6b1 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -6,6 +6,7 @@ import dbt.exceptions import dbt.flags +from dbt import deprecations from dbt.helper_types import PathSet from dbt.include.global_project import PACKAGES from dbt.logger import GLOBAL_LOGGER as logger, DbtProcessState @@ -331,6 +332,9 @@ def load_all( ) -> Manifest: with PARSING_STATE: projects = root_config.load_dependencies() + for project in projects.values(): + if project.config_version == 1: + deprecations.warn('dbt-project-yaml-v1') loader = cls(root_config, projects, macro_hook) loader.load(internal_manifest=internal_manifest) loader.write_parse_results() diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index a872115dc4a..d70ac8f9817 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -2,6 +2,7 @@ from abc import ABCMeta, abstractmethod from typing import Type, Union +from dbt import deprecations from dbt.adapters.factory import register_adapter from dbt.config import RuntimeConfig, Project from dbt.config.profile import read_profile, PROFILES_DIR @@ -132,6 +133,8 @@ class ConfiguredTask(BaseTask): ConfigType = RuntimeConfig def __init__(self, args, config): + if config.config_version == 1: + deprecations.warn('dbt-project-yaml-v1') super().__init__(args, config) register_adapter(self.config) diff --git a/plugins/bigquery/dbt/include/bigquery/dbt_project.yml b/plugins/bigquery/dbt/include/bigquery/dbt_project.yml index edae5386994..b4e88b7b0a4 100644 --- a/plugins/bigquery/dbt/include/bigquery/dbt_project.yml +++ b/plugins/bigquery/dbt/include/bigquery/dbt_project.yml @@ -1,4 +1,4 @@ - +config-version: 2 name: dbt_bigquery version: 1.0 diff --git a/plugins/postgres/dbt/include/postgres/dbt_project.yml b/plugins/postgres/dbt/include/postgres/dbt_project.yml index 266eba33db9..081149f6fd7 100644 --- a/plugins/postgres/dbt/include/postgres/dbt_project.yml +++ b/plugins/postgres/dbt/include/postgres/dbt_project.yml @@ -1,4 +1,4 @@ - +config-version: 2 name: dbt_postgres version: 1.0 diff --git a/plugins/redshift/dbt/include/redshift/dbt_project.yml b/plugins/redshift/dbt/include/redshift/dbt_project.yml index edcd805ab7a..1efdab2c1b0 100644 --- a/plugins/redshift/dbt/include/redshift/dbt_project.yml +++ b/plugins/redshift/dbt/include/redshift/dbt_project.yml @@ -1,4 +1,4 @@ - +config-version: 2 name: dbt_redshift version: 1.0 diff --git a/plugins/snowflake/dbt/include/snowflake/dbt_project.yml b/plugins/snowflake/dbt/include/snowflake/dbt_project.yml index 587a22b5232..fcd2c9a4822 100644 --- a/plugins/snowflake/dbt/include/snowflake/dbt_project.yml +++ b/plugins/snowflake/dbt/include/snowflake/dbt_project.yml @@ -1,4 +1,4 @@ - +config-version: 2 name: dbt_snowflake version: 1.0 diff --git a/test/integration/001_simple_copy_test/test_simple_copy.py b/test/integration/001_simple_copy_test/test_simple_copy.py index 373c7712ff8..bf27171e236 100644 --- a/test/integration/001_simple_copy_test/test_simple_copy.py +++ b/test/integration/001_simple_copy_test/test_simple_copy.py @@ -397,7 +397,7 @@ def postgres_profile(self): @property def project_config(self): - return {} + return {'config-version': 2} @use_profile('postgres') def test_postgres_run_mixed_case(self): diff --git a/test/integration/006_simple_dependency_test/test_local_dependency.py b/test/integration/006_simple_dependency_test/test_local_dependency.py index 5487b5700c0..ec1def2b199 100644 --- a/test/integration/006_simple_dependency_test/test_local_dependency.py +++ b/test/integration/006_simple_dependency_test/test_local_dependency.py @@ -35,6 +35,11 @@ def packages_config(self): ] } + def run_dbt(self, *args, **kwargs): + strict = kwargs.pop('strict', False) + kwargs['strict'] = strict + return super().run_dbt(*args, **kwargs) + class TestSimpleDependency(BaseDependencyTest): @@ -76,7 +81,7 @@ def test_postgres_no_dependency_paths(self): # this should work local_path = os.path.join('local_models', 'my_model.sql') results = self.run_dbt( - ['run', '--models', f'+{local_path}'] + ['run', '--models', f'+{local_path}'], ) # should run the dependency and my_model self.assertEqual(len(results), 2) @@ -103,7 +108,7 @@ def models(self): def test_postgres_missing_dependency(self): # dbt should raise a dbt exception, not raise a parse-time TypeError. with self.assertRaises(dbt.exceptions.Exception) as exc: - self.run_dbt(['compile']) + self.run_dbt(['compile'], strict=False) message = str(exc.exception) self.assertIn('no_such_dependency', message) self.assertIn('is undefined', message) @@ -123,6 +128,11 @@ def project_config(self): return { 'config-version': 2, 'macro-paths': ['schema_override_macros'], + 'models': { + 'config': { + 'schema': 'dbt_test', + }, + }, 'seeds': { 'config': { 'schema': 'dbt_test', @@ -230,6 +240,11 @@ def packages_config(self): ] } + def run_dbt(self, *args, **kwargs): + strict = kwargs.pop('strict', False) + kwargs['strict'] = strict + return super().run_dbt(*args, **kwargs) + @use_profile('postgres') def test_postgres_local_dependency_same_name(self): with self.assertRaises(dbt.exceptions.DependencyException): diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency.py b/test/integration/006_simple_dependency_test/test_simple_dependency.py index a7e64821d28..cf525968593 100644 --- a/test/integration/006_simple_dependency_test/test_simple_dependency.py +++ b/test/integration/006_simple_dependency_test/test_simple_dependency.py @@ -30,6 +30,12 @@ def packages_config(self): ] } + def run_dbt(self, cmd=None, *args, **kwargs): + if cmd and cmd[0] != 'deps': + strict = kwargs.pop('strict', False) + kwargs['strict'] = strict + return super().run_dbt(cmd, *args, **kwargs) + def run_deps(self): return self.run_dbt(["deps"]) @@ -194,7 +200,7 @@ def packages_config(self): def deps_run_assert_equality(self): self.run_dbt(["deps"]) - results = self.run_dbt(["run"]) + results = self.run_dbt(["run"], strict=False) self.assertEqual(len(results), 4) self.assertTablesEqual("seed","table_model") diff --git a/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py b/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py index 4441397b589..ca05c30109a 100644 --- a/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py +++ b/test/integration/006_simple_dependency_test/test_simple_dependency_with_configs.py @@ -15,6 +15,11 @@ def schema(self): def models(self): return "models" + def run_dbt(self, *args, **kwargs): + strict = kwargs.pop('strict', False) + kwargs['strict'] = strict + return super().run_dbt(*args, **kwargs) + class TestSimpleDependencyWithConfigs(BaseTestSimpleDependencyWithConfigs): @property def packages_config(self): diff --git a/test/integration/009_data_tests_test/test_data_tests.py b/test/integration/009_data_tests_test/test_data_tests.py index 23e4271b905..c255c11932d 100644 --- a/test/integration/009_data_tests_test/test_data_tests.py +++ b/test/integration/009_data_tests_test/test_data_tests.py @@ -11,6 +11,7 @@ class TestDataTests(DBTIntegrationTest): @property def project_config(self): return { + 'config-version': 2, "test-paths": [self.test_path] } diff --git a/test/integration/012_deprecation_tests/test_deprecations.py b/test/integration/012_deprecation_tests/test_deprecations.py index ef189063b99..a72815ff855 100644 --- a/test/integration/012_deprecation_tests/test_deprecations.py +++ b/test/integration/012_deprecation_tests/test_deprecations.py @@ -17,12 +17,12 @@ def schema(self): def dir(path): return path.lstrip("/") + +class TestDeprecations(BaseTestDeprecations): @property def models(self): return self.dir("models") - -class TestDeprecations(BaseTestDeprecations): @use_profile('postgres') def test_postgres_deprecations_fail(self): self.run_dbt(strict=True, expect_pass=False) @@ -79,3 +79,29 @@ def test_postgres_deprecations(self): self.run_dbt(strict=False) expected = {'models-key-mismatch'} self.assertEqual(expected, deprecations.active_deprecations) + + +class TestDbtProjectYamlV1Deprecation(BaseTestDeprecations): + @property + def models(self): + return 'boring-models' + + @property + def project_config(self): + # No config-version set! + return {} + + @use_profile('postgres') + def test_postgres_project_deprecations_fail(self): + with self.assertRaises(dbt.exceptions.CompilationException) as exc: + self.run_dbt(strict=True) + + exc_str = ' '.join(str(exc.exception).split()) # flatten all whitespace + self.assertIn('dbt_project.yml has been upgraded to config version 2', exc_str) + + @use_profile('postgres') + def test_postgres_project_deprecations(self): + self.assertEqual(deprecations.active_deprecations, set()) + self.run_dbt(strict=False) + expected = {'dbt-project-yaml-v1'} + self.assertEqual(expected, deprecations.active_deprecations) diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 2605746c641..6229c016ff8 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -1525,7 +1525,6 @@ def expected_postgres_references_manifest(self, model_database=None): }, 'config': { 'enabled': True, - 'quoting': {}, }, 'quoting': { 'database': False, diff --git a/test/integration/034_redshift_test/test_late_binding_view.py b/test/integration/034_redshift_test/test_late_binding_view.py index fdb0194e0aa..656e1c2d801 100644 --- a/test/integration/034_redshift_test/test_late_binding_view.py +++ b/test/integration/034_redshift_test/test_late_binding_view.py @@ -19,9 +19,12 @@ def models(self): @property def project_config(self): return { + 'config-version': 2, 'data-paths': [self.dir('seed')], 'seeds': { - 'quote_columns': False, + 'config': { + 'quote_columns': False, + } } } diff --git a/test/integration/040_override_database_test/test_override_database.py b/test/integration/040_override_database_test/test_override_database.py index a07652ca22d..c3cf1130ec2 100644 --- a/test/integration/040_override_database_test/test_override_database.py +++ b/test/integration/040_override_database_test/test_override_database.py @@ -109,17 +109,22 @@ def run_database_override(self): func = lambda x: x self.use_default_project({ + 'config-version': 2, + 'vars': { + 'alternate_db': self.alternative_database, + }, 'models': { - 'vars': { - 'alternate_db': self.alternative_database, + 'config': { + 'database': self.alternative_database, }, - 'database': self.alternative_database, 'test': { 'subfolder': { - 'database': self.default_database, - }, - }, - } + 'config': { + 'database': self.default_database, + } + } + } + }, }) self.run_dbt_notstrict(['seed']) @@ -149,7 +154,12 @@ def run_database_override(self): func = lambda x: x self.use_default_project({ - 'seeds': {'database': self.alternative_database} + 'config-version': 2, + 'seeds': { + 'config': { + 'database': self.alternative_database + }, + }, }) self.run_dbt_notstrict(['seed']) diff --git a/test/integration/047_dbt_ls_test/test_ls.py b/test/integration/047_dbt_ls_test/test_ls.py index e066d9fc975..d1e5e5123dc 100644 --- a/test/integration/047_dbt_ls_test/test_ls.py +++ b/test/integration/047_dbt_ls_test/test_ls.py @@ -225,7 +225,6 @@ def expect_source_output(self): 'json': { 'config': { 'enabled': True, - 'quoting': {}, }, 'package_name': 'test', 'name': 'my_table', diff --git a/test/integration/048_rpc_test/test_rpc.py b/test/integration/048_rpc_test/test_rpc.py index 6d45d91fff4..e981ccfa256 100644 --- a/test/integration/048_rpc_test/test_rpc.py +++ b/test/integration/048_rpc_test/test_rpc.py @@ -22,7 +22,7 @@ class ServerProcess(dbt.flags.MP_CONTEXT.Process): def __init__(self, port, profiles_dir, cli_vars=None): self.port = port handle_and_check_args = [ - '--strict', 'rpc', '--log-cache-events', + 'rpc', '--log-cache-events', '--port', str(self.port), '--profiles-dir', profiles_dir ] diff --git a/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml b/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml index 2c58789b48e..248abd809ac 100644 --- a/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml +++ b/test/integration/053_custom_materialization/override-view-adapter-dep/dbt_project.yml @@ -1,3 +1,4 @@ name: view_adapter_override version: '1.0' macro-paths: ['macros'] +config-version: 2 diff --git a/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml b/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml index 2c58789b48e..248abd809ac 100644 --- a/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml +++ b/test/integration/053_custom_materialization/override-view-adapter-pass-dep/dbt_project.yml @@ -1,3 +1,4 @@ name: view_adapter_override version: '1.0' macro-paths: ['macros'] +config-version: 2 diff --git a/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml b/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml index 9b1515079d2..f8fe48084c0 100644 --- a/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml +++ b/test/integration/053_custom_materialization/override-view-default-dep/dbt_project.yml @@ -1,3 +1,4 @@ name: view_default_override +config-version: 2 version: '1.0' macro-paths: ['macros'] diff --git a/test/rpc/test_base.py b/test/rpc/test_base.py index e94f86d49a6..e551a659fc5 100644 --- a/test/rpc/test_base.py +++ b/test/rpc/test_base.py @@ -601,7 +601,7 @@ def test_rpc_seed_threads( project_root, profiles_root, postgres_profile, unique_schema ): project = ProjectDefinition( - project_data={'seeds': {'quote_columns': False}}, + project_data={'seeds': {'config': {'quote_columns': False}}}, seeds={'data.csv': 'a,b\n1,hello\n2,goodbye'}, ) querier_ctx = get_querier( @@ -626,7 +626,7 @@ def test_rpc_seed_include_exclude( project_root, profiles_root, postgres_profile, unique_schema ): project = ProjectDefinition( - project_data={'seeds': {'quote_columns': False}}, + project_data={'seeds': {'config': {'quote_columns': False}}}, seeds={ 'data_1.csv': 'a,b\n1,hello\n2,goodbye', 'data_2.csv': 'a,b\n1,data', @@ -773,7 +773,7 @@ def test_source_freshness( error_me = start_time - timedelta(days=2) # this should trigger a 'warn' project = ProjectDefinition( - project_data={'seeds': {'quote_columns': False}}, + project_data={'seeds': {'config': {'quote_columns': False}}}, seeds={ 'source.csv': 'a,b\n1,{}\n'.format(error_me.strftime('%Y-%m-%d %H:%M:%S')), 'other_source.csv': 'a,b\n1,{}\n'.format(error_me.strftime('%Y-%m-%d %H:%M:%S')) diff --git a/test/rpc/util.py b/test/rpc/util.py index c684238f95c..86c62c5c536 100644 --- a/test/rpc/util.py +++ b/test/rpc/util.py @@ -37,7 +37,7 @@ def __init__( self.criteria = criteria self.error = None handle_and_check_args = [ - '--strict', 'rpc', '--log-cache-events', + 'rpc', '--log-cache-events', '--port', str(self.port), '--profiles-dir', profiles_dir ] @@ -473,6 +473,7 @@ def __init__( seeds=None, ): self.project = { + 'config-version': 2, 'name': name, 'version': version, 'profile': profile, diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index d2ddf9a366d..dace3716bbb 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -1323,7 +1323,6 @@ def test_basic(self): 'tags': [], 'config': { 'enabled': True, - 'quoting': {}, } } source_def = self.ContractType(