From 2599f0393edf378ca0ff1319fcc4ceadcc614e76 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 27 Feb 2023 09:08:22 -0600 Subject: [PATCH 01/14] allow optional config-version --- core/dbt/config/project.py | 2 +- core/dbt/contracts/project.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 8fc115e3820..fac991c6b47 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -499,7 +499,7 @@ def from_project_root( ) -> "PartialProject": project_root = os.path.normpath(project_root) project_dict = load_raw_project(project_root) - config_version = project_dict.get("config-version", 1) + config_version = project_dict.get("config-version", 2) if config_version != 2: raise DbtProjectError( f"Invalid config version: {config_version}, expected 2", diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 3041b1bd4b9..18c6a0ba758 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -184,7 +184,7 @@ class RegistryPackageMetadata( @dataclass class Project(HyphenatedDbtClassMixin, Replaceable): name: Identifier - config_version: int + config_version: Optional[int] = 2 version: Optional[Union[SemverString, float]] = None project_root: Optional[str] = None source_paths: Optional[List[str]] = None From 7d668047e30245a55e01db50396cfd3ac35b0a03 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 27 Feb 2023 09:09:28 -0600 Subject: [PATCH 02/14] remove schema yml version missing error --- core/dbt/exceptions.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index e2db02f6d1b..ac358e7f5b6 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -2078,13 +2078,6 @@ def get_message(self) -> str: return msg -class PropertyYMLMissingVersionError(PropertyYMLError): - def __init__(self, path: str): - self.path = path - self.issue = f"the yml property file {self.path} is missing a version tag" - super().__init__(self.path, self.issue) - - class PropertyYMLVersionNotIntError(PropertyYMLError): def __init__(self, path: str, version: Any): self.path = path From caa0c9d87650f67296301c2e5cea361096db9ac3 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 27 Feb 2023 09:09:57 -0600 Subject: [PATCH 03/14] mkae version optional, default 2 --- core/dbt/parser/schemas.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index cfeb2bc8aef..2a210339763 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -63,7 +63,6 @@ TestConfigError, ParsingError, PropertyYMLInvalidTagError, - PropertyYMLMissingVersionError, PropertyYMLVersionNotIntError, DbtValidationError, YamlLoadError, @@ -576,10 +575,7 @@ def parse_file(self, block: FileBlock, dct: Dict = None) -> None: def check_format_version(file_path, yaml_dct) -> None: - if "version" not in yaml_dct: - raise PropertyYMLMissingVersionError(file_path) - - version = yaml_dct["version"] + version = yaml_dct.get("version", 2) # if it's not an integer, the version is malformed, or not # set. Either way, only 'version: 2' is supported. if not isinstance(version, int): From 5c08e50dd323458041397b94348773c988a642eb Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 27 Feb 2023 09:10:46 -0600 Subject: [PATCH 04/14] add tests for config-version and schema yml version --- core/dbt/tests/fixtures/project.py | 11 ++++++++- tests/functional/basic/test_project.py | 33 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 4bb9b81369a..d4fb352bd2c 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -174,10 +174,16 @@ def project_config_update(): return {} +# Data used to remove keys from project configs +@pytest.fixture(scope="class") +def project_config_remove(): + return [] + + # Combines the project_config_update dictionary with project_config defaults to # produce a project_yml config and write it out as dbt_project.yml @pytest.fixture(scope="class") -def dbt_project_yml(project_root, project_config_update, logs_dir): +def dbt_project_yml(project_root, project_config_update, project_config_remove, logs_dir): project_config = { "config-version": 2, "name": "test", @@ -190,6 +196,9 @@ def dbt_project_yml(project_root, project_config_update, logs_dir): elif isinstance(project_config_update, str): updates = yaml.safe_load(project_config_update) project_config.update(updates) + if project_config_remove: + for key in project_config_remove: + project_config.pop(key, None) write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") return project_config diff --git a/tests/functional/basic/test_project.py b/tests/functional/basic/test_project.py index 77f01c97b9f..10427c5ec3b 100644 --- a/tests/functional/basic/test_project.py +++ b/tests/functional/basic/test_project.py @@ -3,6 +3,39 @@ from dbt.exceptions import ProjectContractError +simple_model_sql = """ +select true as my_column +""" + +simple_model_yml = """ +models: + - name: simple_model + description: "is sythentic data ok? my column:" + columns: + - name: my_column + description: asked and answered +""" + + +class TestSchemaYmlVersionMissing: + @pytest.fixture(scope="class") + def models(self): + return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml} + + def test_empty_version(self, project): + run_dbt(["run"], expect_pass=True) + + +class TestProjectConfigVersionMissing: + # default dbt_project.yml has config-version: 2 + @pytest.fixture(scope="class") + def project_config_remove(self): + return ["config-version"] + + def test_empty_version(self, project): + run_dbt(["run"], expect_pass=True) + + class TestProjectYamlVersionMissing: # default dbt_project.yml does not fill version From 12fc8b027c8e38e5d4f875c946a093b47b7c534b Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 27 Feb 2023 09:13:30 -0600 Subject: [PATCH 05/14] say changie --- .changes/unreleased/Features-20230227-091316.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20230227-091316.yaml diff --git a/.changes/unreleased/Features-20230227-091316.yaml b/.changes/unreleased/Features-20230227-091316.yaml new file mode 100644 index 00000000000..264c7960216 --- /dev/null +++ b/.changes/unreleased/Features-20230227-091316.yaml @@ -0,0 +1,6 @@ +kind: Features +body: make version configs optional +time: 2023-02-27T09:13:16.104386-06:00 +custom: + Author: dave-connors-3 + Issue: "7054" From db84daae5a991d0d1cc87c698a1926ea7d390479 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 27 Feb 2023 09:27:11 -0600 Subject: [PATCH 06/14] update old test with non int value --- test/unit/test_contracts_project.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/test_contracts_project.py b/test/unit/test_contracts_project.py index c100eb24fcc..401e3fdc7e9 100644 --- a/test/unit/test_contracts_project.py +++ b/test/unit/test_contracts_project.py @@ -42,6 +42,7 @@ def test_unsupported_version(self): 'version': '1.0', 'profile': 'test', 'project-root': '/usr/src/app', + 'config-version': 'non-int', } with self.assertRaises(Exception): self.ContractType.from_dict(dct) From 53c3a8297c2c61c8383370af9dd5854f73a8f709 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:04:01 -0600 Subject: [PATCH 07/14] remove get method and version check for config-version from project.py --- core/dbt/config/project.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index fac991c6b47..f0d3d94e3f6 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -499,13 +499,6 @@ def from_project_root( ) -> "PartialProject": project_root = os.path.normpath(project_root) project_dict = load_raw_project(project_root) - config_version = project_dict.get("config-version", 2) - if config_version != 2: - raise DbtProjectError( - f"Invalid config version: {config_version}, expected 2", - path=os.path.join(project_root, "dbt_project.yml"), - ) - packages_dict = package_data_from_root(project_root) selectors_dict = selector_data_from_root(project_root) return cls.from_dicts( From 733789d667dc825a72cfb6d46fe197664f6164f4 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:06:35 -0600 Subject: [PATCH 08/14] remove check_format_version --- core/dbt/parser/schemas.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 2a210339763..5f54b2d6d1c 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -62,8 +62,6 @@ SchemaConfigError, TestConfigError, ParsingError, - PropertyYMLInvalidTagError, - PropertyYMLVersionNotIntError, DbtValidationError, YamlLoadError, YamlParseDictError, @@ -574,17 +572,6 @@ def parse_file(self, block: FileBlock, dct: Dict = None) -> None: group_parser.parse() -def check_format_version(file_path, yaml_dct) -> None: - version = yaml_dct.get("version", 2) - # if it's not an integer, the version is malformed, or not - # set. Either way, only 'version: 2' is supported. - if not isinstance(version, int): - raise PropertyYMLVersionNotIntError(file_path, version) - - if version != 2: - raise PropertyYMLInvalidTagError(file_path, version) - - Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch) NodeTarget = TypeVar("NodeTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate) NonSourceTarget = TypeVar( From 681a76ce4643480d26e605d71986624f920984dd Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:07:33 -0600 Subject: [PATCH 09/14] deprecate schema version exceptions --- core/dbt/exceptions.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index ac358e7f5b6..e128369fa76 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -2078,25 +2078,6 @@ def get_message(self) -> str: return msg -class PropertyYMLVersionNotIntError(PropertyYMLError): - def __init__(self, path: str, version: Any): - self.path = path - self.version = version - self.issue = ( - "its 'version:' tag must be an integer (e.g. version: 2)." - f" {self.version} is not an integer" - ) - super().__init__(self.path, self.issue) - - -class PropertyYMLInvalidTagError(PropertyYMLError): - def __init__(self, path: str, version: int): - self.path = path - self.version = version - self.issue = f"its 'version:' tag is set to {self.version}. Only 2 is supported" - super().__init__(self.path, self.issue) - - class RelationWrongTypeError(CompilationError): def __init__(self, relation, expected_type, model=None): self.relation = relation From d8190d5e03e80c63eefec1b9ae46e44a696b8d43 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:10:07 -0600 Subject: [PATCH 10/14] revert the pop method logic changes --- core/dbt/tests/fixtures/project.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index d4fb352bd2c..4bb9b81369a 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -174,16 +174,10 @@ def project_config_update(): return {} -# Data used to remove keys from project configs -@pytest.fixture(scope="class") -def project_config_remove(): - return [] - - # Combines the project_config_update dictionary with project_config defaults to # produce a project_yml config and write it out as dbt_project.yml @pytest.fixture(scope="class") -def dbt_project_yml(project_root, project_config_update, project_config_remove, logs_dir): +def dbt_project_yml(project_root, project_config_update, logs_dir): project_config = { "config-version": 2, "name": "test", @@ -196,9 +190,6 @@ def dbt_project_yml(project_root, project_config_update, project_config_remove, elif isinstance(project_config_update, str): updates = yaml.safe_load(project_config_update) project_config.update(updates) - if project_config_remove: - for key in project_config_remove: - project_config.pop(key, None) write_file(yaml.safe_dump(project_config), project_root, "dbt_project.yml") return project_config From 2d0a6874d3c3dc405184abdb311b9132c58e5211 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:13:34 -0600 Subject: [PATCH 11/14] remove config-version from global project config dict test fixture --- core/dbt/tests/fixtures/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/tests/fixtures/project.py b/core/dbt/tests/fixtures/project.py index 4bb9b81369a..bfdb4dfeab7 100644 --- a/core/dbt/tests/fixtures/project.py +++ b/core/dbt/tests/fixtures/project.py @@ -179,7 +179,6 @@ def project_config_update(): @pytest.fixture(scope="class") def dbt_project_yml(project_root, project_config_update, logs_dir): project_config = { - "config-version": 2, "name": "test", "profile": "test", "log-path": logs_dir, From a63da5a64041a981ab54a4bb34b74b2bc43be543 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:14:01 -0600 Subject: [PATCH 12/14] revert non-int config-version in test --- test/unit/test_contracts_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_contracts_project.py b/test/unit/test_contracts_project.py index 401e3fdc7e9..4e5070a4f8b 100644 --- a/test/unit/test_contracts_project.py +++ b/test/unit/test_contracts_project.py @@ -1,3 +1,4 @@ + from .utils import ContractTestCase from dbt.dataclass_schema import ValidationError @@ -42,7 +43,6 @@ def test_unsupported_version(self): 'version': '1.0', 'profile': 'test', 'project-root': '/usr/src/app', - 'config-version': 'non-int', } with self.assertRaises(Exception): self.ContractType.from_dict(dct) From 40db0c3c861a3a1d18236078cc03f151fb7ef290 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:22:56 -0600 Subject: [PATCH 13/14] remove references to old method --- core/dbt/parser/read_files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/dbt/parser/read_files.py b/core/dbt/parser/read_files.py index 531e5f39560..8683f7c8e69 100644 --- a/core/dbt/parser/read_files.py +++ b/core/dbt/parser/read_files.py @@ -11,7 +11,7 @@ SchemaSourceFile, ) -from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version +from dbt.parser.schemas import yaml_from_file, schema_file_keys from dbt.exceptions import ParsingError from dbt.parser.search import filesystem_search from typing import Optional @@ -67,7 +67,6 @@ def load_source_file( # Check version, that key values are lists and that each element in # the lists has a 'name' key def validate_yaml(file_path, dct): - check_format_version(file_path, dct) for key in schema_file_keys: if key in dct: if not isinstance(dct[key], list): From 5b7ffc4cbf0df58812d2609db875b8223c9b5d59 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 9 Mar 2023 14:23:53 -0600 Subject: [PATCH 14/14] remove test_unsupported_version --- test/unit/test_contracts_project.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/unit/test_contracts_project.py b/test/unit/test_contracts_project.py index 4e5070a4f8b..edb206ac202 100644 --- a/test/unit/test_contracts_project.py +++ b/test/unit/test_contracts_project.py @@ -36,13 +36,3 @@ def test_invalid_name(self): } with self.assertRaises(ValidationError): self.ContractType.validate(dct) - - def test_unsupported_version(self): - dct = { - 'name': 'test', - 'version': '1.0', - 'profile': 'test', - 'project-root': '/usr/src/app', - } - with self.assertRaises(Exception): - self.ContractType.from_dict(dct)