diff --git a/tests/unit/clients/__init__.py b/tests/unit/clients/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_jinja.py b/tests/unit/clients/test_jinja.py similarity index 85% rename from tests/unit/test_jinja.py rename to tests/unit/clients/test_jinja.py index 94150be8e26..5f15d9e3f44 100644 --- a/tests/unit/test_jinja.py +++ b/tests/unit/clients/test_jinja.py @@ -1,4 +1,3 @@ -import unittest from contextlib import contextmanager import pytest @@ -367,7 +366,7 @@ def expected_id(arg): @pytest.mark.parametrize("value,text_expectation,native_expectation", jinja_tests, ids=expected_id) -def test_jinja_rendering(value, text_expectation, native_expectation): +def test_jinja_rendering_string(value, text_expectation, native_expectation): foo_value = yaml.safe_load(value)["foo"] ctx = {"a_str": "100", "a_int": 100, "b_str": "hello"} with text_expectation as text_result: @@ -377,40 +376,41 @@ def test_jinja_rendering(value, text_expectation, native_expectation): assert native_result == get_rendered(foo_value, ctx, native=True) -class TestJinja(unittest.TestCase): - def test_do(self): - s = "{% set my_dict = {} %}\n{% do my_dict.update(a=1) %}" +def test_do(): + s = "{% set my_dict = {} %}\n{% do my_dict.update(a=1) %}" - template = get_template(s, {}) - mod = template.make_module() - self.assertEqual(mod.my_dict, {"a": 1}) + template = get_template(s, {}) + mod = template.make_module() + assert mod.my_dict == {"a": 1} - def test_regular_render(self): - s = '{{ "some_value" | as_native }}' - value = get_rendered(s, {}, native=False) - assert value == "some_value" - s = "{{ 1991 | as_native }}" - value = get_rendered(s, {}, native=False) - assert value == "1991" - s = '{{ "some_value" | as_text }}' - value = get_rendered(s, {}, native=False) - assert value == "some_value" - s = "{{ 1991 | as_text }}" - value = get_rendered(s, {}, native=False) - assert value == "1991" +def test_regular_render(): + s = '{{ "some_value" | as_native }}' + value = get_rendered(s, {}, native=False) + assert value == "some_value" + s = "{{ 1991 | as_native }}" + value = get_rendered(s, {}, native=False) + assert value == "1991" - def test_native_render(self): - s = '{{ "some_value" | as_native }}' - value = get_rendered(s, {}, native=True) - assert value == "some_value" - s = "{{ 1991 | as_native }}" - value = get_rendered(s, {}, native=True) - assert value == 1991 + s = '{{ "some_value" | as_text }}' + value = get_rendered(s, {}, native=False) + assert value == "some_value" + s = "{{ 1991 | as_text }}" + value = get_rendered(s, {}, native=False) + assert value == "1991" - s = '{{ "some_value" | as_text }}' - value = get_rendered(s, {}, native=True) - assert value == "some_value" - s = "{{ 1991 | as_text }}" - value = get_rendered(s, {}, native=True) - assert value == "1991" + +def test_native_render(): + s = '{{ "some_value" | as_native }}' + value = get_rendered(s, {}, native=True) + assert value == "some_value" + s = "{{ 1991 | as_native }}" + value = get_rendered(s, {}, native=True) + assert value == 1991 + + s = '{{ "some_value" | as_text }}' + value = get_rendered(s, {}, native=True) + assert value == "some_value" + s = "{{ 1991 | as_text }}" + value = get_rendered(s, {}, native=True) + assert value == "1991" diff --git a/tests/unit/test_macro_calls.py b/tests/unit/clients/test_jinja_static.py similarity index 100% rename from tests/unit/test_macro_calls.py rename to tests/unit/clients/test_jinja_static.py diff --git a/tests/unit/test_registry_get_request_exception.py b/tests/unit/clients/test_registry.py similarity index 100% rename from tests/unit/test_registry_get_request_exception.py rename to tests/unit/clients/test_registry.py diff --git a/tests/unit/config/test_selectors.py b/tests/unit/config/test_selectors.py index 9688647a00e..d306fb55282 100644 --- a/tests/unit/config/test_selectors.py +++ b/tests/unit/config/test_selectors.py @@ -4,7 +4,8 @@ import yaml import dbt.exceptions -from dbt.config.selectors import SelectorConfig, selector_config_from_data +from dbt.config.selectors import SelectorConfig, SelectorDict, selector_config_from_data +from dbt.exceptions import DbtSelectorsError def get_selector_dict(txt: str) -> dict: @@ -201,3 +202,188 @@ def test_multiple_default_true(self): dbt.exceptions.DbtSelectorsError, "Found multiple selectors with `default: true`:" ): selector_config_from_data(dct) + + def test_compare_cli_non_cli(self): + dct = get_selector_dict( + """\ + selectors: + - name: nightly_diet_snowplow + description: "This uses more CLI-style syntax" + definition: + union: + - intersection: + - '@source:snowplow' + - 'tag:nightly' + - 'models/export' + - exclude: + - intersection: + - 'package:snowplow' + - 'config.materialized:incremental' + - export_performance_timing + - name: nightly_diet_snowplow_full + description: "This is a fuller YAML specification" + definition: + union: + - intersection: + - method: source + value: snowplow + childrens_parents: true + - method: tag + value: nightly + - method: path + value: models/export + - exclude: + - intersection: + - method: package + value: snowplow + - method: config.materialized + value: incremental + - method: fqn + value: export_performance_timing + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + with_strings = sel_dict["nightly_diet_snowplow"]["definition"] + no_strings = sel_dict["nightly_diet_snowplow_full"]["definition"] + self.assertEqual(with_strings, no_strings) + + def test_single_string_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: nightly_selector + definition: + 'tag:nightly' + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "tag", "value": "nightly"} + definition = sel_dict["nightly_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_single_key_value_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: nightly_selector + definition: + tag: nightly + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "tag", "value": "nightly"} + definition = sel_dict["nightly_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_parent_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: kpi_nightly_selector + definition: + '+exposure:kpi_nightly' + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "exposure", "value": "kpi_nightly", "parents": True} + definition = sel_dict["kpi_nightly_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_plus_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: my_model_children_selector + definition: + 'my_model+2' + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "fqn", "value": "my_model", "children": True, "children_depth": "2"} + definition = sel_dict["my_model_children_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_selector_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: default + definition: + union: + - intersection: + - tag: foo + - tag: bar + - name: inherited + definition: + method: selector + value: default + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + definition = sel_dict["default"]["definition"] + expected = sel_dict["inherited"]["definition"] + self.assertEqual(expected, definition) + + def test_selector_definition_with_exclusion(self): + dct = get_selector_dict( + """\ + selectors: + - name: default + definition: + union: + - intersection: + - tag: foo + - tag: bar + - name: inherited + definition: + union: + - method: selector + value: default + - exclude: + - tag: bar + - name: comparison + definition: + union: + - union: + - intersection: + - tag: foo + - tag: bar + - exclude: + - tag: bar + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list((dct["selectors"])) + assert sel_dict + definition = sel_dict["inherited"]["definition"] + expected = sel_dict["comparison"]["definition"] + self.assertEqual(expected, definition) + + def test_missing_selector(self): + dct = get_selector_dict( + """\ + selectors: + - name: inherited + definition: + method: selector + value: default + """ + ) + with self.assertRaises(DbtSelectorsError) as err: + SelectorDict.parse_from_selectors_list((dct["selectors"])) + + self.assertEqual( + "Existing selector definition for default not found.", str(err.exception.msg) + ) diff --git a/tests/unit/context/__init__.py b/tests/unit/context/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_base_context.py b/tests/unit/context/test_base.py similarity index 100% rename from tests/unit/test_base_context.py rename to tests/unit/context/test_base.py diff --git a/tests/unit/test_contracts_graph_node_args.py b/tests/unit/contracts/graph/test_node_args.py similarity index 100% rename from tests/unit/test_contracts_graph_node_args.py rename to tests/unit/contracts/graph/test_node_args.py diff --git a/tests/unit/test_contracts_graph_compiled.py b/tests/unit/contracts/graph/test_nodes.py similarity index 72% rename from tests/unit/test_contracts_graph_compiled.py rename to tests/unit/contracts/graph/test_nodes.py index e0228cc28f6..a498b99dcbc 100644 --- a/tests/unit/test_contracts_graph_compiled.py +++ b/tests/unit/contracts/graph/test_nodes.py @@ -1,15 +1,22 @@ import pickle +import re from dataclasses import replace import pytest from dbt.artifacts.resources import ColumnInfo, TestConfig, TestMetadata +from dbt.compilation import inject_ctes_into_sql from dbt.contracts.files import FileHash -from dbt.contracts.graph.nodes import DependsOn, GenericTestNode, ModelConfig, ModelNode +from dbt.contracts.graph.nodes import ( + DependsOn, + GenericTestNode, + InjectedCTE, + ModelConfig, + ModelNode, +) from dbt.node_types import NodeType from tests.unit.fixtures import generic_test_node, model_node - -from .utils import ( +from tests.unit.utils import ( assert_fails_validation, assert_from_dict, assert_symmetric, @@ -17,6 +24,12 @@ ) +def norm_whitespace(string): + _RE_COMBINE_WHITESPACE = re.compile(r"\s+") + string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() + return string + + @pytest.fixture def basic_uncompiled_model(): return ModelNode( @@ -603,3 +616,191 @@ def test_compare_to_compiled(basic_uncompiled_schema_test_node, basic_compiled_s compiled, config=fixed_config, unrendered_config=uncompiled.unrendered_config ) assert uncompiled.same_contents(fixed_compiled, "postgres") + + +def test_inject_ctes_simple1(): + starting_sql = "select * from __dbt__cte__base" + ctes = [ + InjectedCTE( + id="model.test.base", + sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", + ) + ] + expected_sql = """with __dbt__cte__base as ( + select * from test16873767336887004702_test_ephemeral.seed + ) select * from __dbt__cte__base""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_simple2(): + starting_sql = "select * from __dbt__cte__ephemeral_level_two" + ctes = [ + InjectedCTE( + id="model.test.ephemeral_level_two", + sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873757769710148165_test_ephemeral"."source_table"\n)', + ) + ] + expected_sql = """with __dbt__cte__ephemeral_level_two as ( + select * from "dbt"."test16873757769710148165_test_ephemeral"."source_table" + ) select * from __dbt__cte__ephemeral_level_two""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_multiple_ctes(): + + starting_sql = "select * from __dbt__cte__ephemeral" + ctes = [ + InjectedCTE( + id="model.test.ephemeral_level_two", + sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873735573223965828_test_ephemeral"."source_table"\n)', + ), + InjectedCTE( + id="model.test.ephemeral", + sql=" __dbt__cte__ephemeral as (\n\nselect * from __dbt__cte__ephemeral_level_two\n)", + ), + ] + expected_sql = """with __dbt__cte__ephemeral_level_two as ( + select * from "dbt"."test16873735573223965828_test_ephemeral"."source_table" + ), __dbt__cte__ephemeral as ( + select * from __dbt__cte__ephemeral_level_two + ) select * from __dbt__cte__ephemeral""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_multiple_ctes_more_complex(): + starting_sql = """select * from __dbt__cte__female_only + union all + select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" + ctes = [ + InjectedCTE( + id="model.test.base", + sql=" __dbt__cte__base as (\n\n\nselect * from test16873757723266827902_test_ephemeral.seed\n)", + ), + InjectedCTE( + id="model.test.base_copy", + sql=" __dbt__cte__base_copy as (\n\n\nselect * from __dbt__cte__base\n)", + ), + InjectedCTE( + id="model.test.female_only", + sql=" __dbt__cte__female_only as (\n\n\nselect * from __dbt__cte__base_copy where gender = 'Female'\n)", + ), + ] + expected_sql = """with __dbt__cte__base as ( + select * from test16873757723266827902_test_ephemeral.seed + ), __dbt__cte__base_copy as ( + select * from __dbt__cte__base + ), __dbt__cte__female_only as ( + select * from __dbt__cte__base_copy where gender = 'Female' + ) select * from __dbt__cte__female_only + union all + select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_starting_with1(): + starting_sql = """ + with internal_cte as (select * from sessions) + select * from internal_cte + """ + ctes = [ + InjectedCTE( + id="cte_id_1", + sql="__dbt__cte__ephemeral as (select * from table)", + ), + InjectedCTE( + id="cte_id_2", + sql="__dbt__cte__events as (select id, type from events)", + ), + ] + expected_sql = """with __dbt__cte__ephemeral as (select * from table), + __dbt__cte__events as (select id, type from events), + internal_cte as (select * from sessions) + select * from internal_cte""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_starting_with2(): + starting_sql = """with my_other_cool_cte as ( + select id, name from __dbt__cte__ephemeral + where id > 1000 + ) + select name, id from my_other_cool_cte""" + ctes = [ + InjectedCTE( + id="model.singular_tests_ephemeral.ephemeral", + sql=' __dbt__cte__ephemeral as (\n\n\nwith my_cool_cte as (\n select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base"\n)\nselect id, name from my_cool_cte where id is not null\n)', + ) + ] + expected_sql = """with __dbt__cte__ephemeral as ( + with my_cool_cte as ( + select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base" + ) + select id, name from my_cool_cte where id is not null + ), my_other_cool_cte as ( + select id, name from __dbt__cte__ephemeral + where id > 1000 + ) + select name, id from my_other_cool_cte""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_comment_with(): + # Test injection with a comment containing "with" + starting_sql = """ + --- This is sql with a comment + select * from __dbt__cte__base + """ + ctes = [ + InjectedCTE( + id="model.test.base", + sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", + ) + ] + expected_sql = """with __dbt__cte__base as ( + select * from test16873767336887004702_test_ephemeral.seed + ) --- This is sql with a comment + select * from __dbt__cte__base""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_with_recursive(): + # Test injection with "recursive" keyword + starting_sql = """ + with recursive t(n) as ( + select * from __dbt__cte__first_ephemeral_model + union all + select n+1 from t where n < 100 + ) + select sum(n) from t + """ + ctes = [ + InjectedCTE( + id="model.test.first_ephemeral_model", + sql=" __dbt__cte__first_ephemeral_model as (\n\nselect 1 as fun\n)", + ) + ] + expected_sql = """with recursive __dbt__cte__first_ephemeral_model as ( + select 1 as fun + ), t(n) as ( + select * from __dbt__cte__first_ephemeral_model + union all + select n+1 from t where n < 100 + ) + select sum(n) from t + """ + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) diff --git a/tests/unit/test_contracts_graph_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py similarity index 99% rename from tests/unit/test_contracts_graph_parsed.py rename to tests/unit/contracts/graph/test_nodes_parsed.py index b94271fab08..4e8392cb3df 100644 --- a/tests/unit/test_contracts_graph_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -55,8 +55,7 @@ ) from dbt.node_types import AccessType, NodeType from dbt_common.dataclass_schema import ValidationError - -from .utils import ( +from tests.unit.utils import ( ContractTestCase, assert_fails_validation, assert_from_dict, diff --git a/tests/unit/contracts/graph/test_unparsed.py b/tests/unit/contracts/graph/test_unparsed.py index 65d29ad0947..90fa2fcf8a7 100644 --- a/tests/unit/contracts/graph/test_unparsed.py +++ b/tests/unit/contracts/graph/test_unparsed.py @@ -1,8 +1,991 @@ +import pickle +from datetime import timedelta + import pytest -from dbt.contracts.graph.unparsed import HasColumnTests, UnparsedColumn +from dbt.artifacts.resources import ( + ExposureType, + FreshnessThreshold, + MaturityType, + Owner, + Quoting, + Time, +) +from dbt.artifacts.resources.types import TimePeriod +from dbt.artifacts.schemas.results import FreshnessStatus +from dbt.contracts.graph.unparsed import ( + Docs, + HasColumnTests, + UnparsedColumn, + UnparsedDocumentationFile, + UnparsedExposure, + UnparsedMacro, + UnparsedMetric, + UnparsedMetricInputMeasure, + UnparsedMetricTypeParams, + UnparsedModelUpdate, + UnparsedNode, + UnparsedNodeUpdate, + UnparsedRunHook, + UnparsedSourceDefinition, + UnparsedSourceTableDefinition, + UnparsedVersion, +) from dbt.exceptions import ParsingError +from dbt.node_types import NodeType from dbt.parser.schemas import ParserRef +from tests.unit.utils import ContractTestCase + + +class TestUnparsedMacro(ContractTestCase): + ContractType = UnparsedMacro + + def test_ok(self): + macro_dict = { + "path": "/root/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", + "resource_type": "macro", + } + macro = self.ContractType( + path="/root/path.sql", + original_file_path="/root/path.sql", + package_name="test", + language="sql", + raw_code="{% macro foo() %}select 1 as id{% endmacro %}", + resource_type=NodeType.Macro, + ) + self.assert_symmetric(macro, macro_dict) + pickle.loads(pickle.dumps(macro)) + + def test_invalid_missing_field(self): + macro_dict = { + "path": "/root/path.sql", + "original_file_path": "/root/path.sql", + # 'package_name': 'test', + "language": "sql", + "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", + "resource_type": "macro", + } + self.assert_fails_validation(macro_dict) + + def test_invalid_extra_field(self): + macro_dict = { + "path": "/root/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", + "extra": "extra", + "resource_type": "macro", + } + self.assert_fails_validation(macro_dict) + + +class TestUnparsedNode(ContractTestCase): + ContractType = UnparsedNode + + def test_ok(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Model, + "path": "/root/x/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": 'select * from {{ ref("thing") }}', + } + node = self.ContractType( + package_name="test", + path="/root/x/path.sql", + original_file_path="/root/path.sql", + language="sql", + raw_code='select * from {{ ref("thing") }}', + name="foo", + resource_type=NodeType.Model, + ) + self.assert_symmetric(node, node_dict) + self.assertFalse(node.empty) + + self.assert_fails_validation(node_dict, cls=UnparsedRunHook) + self.assert_fails_validation(node_dict, cls=UnparsedMacro) + pickle.loads(pickle.dumps(node)) + + def test_empty(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Model, + "path": "/root/x/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": " \n", + } + node = UnparsedNode( + package_name="test", + path="/root/x/path.sql", + original_file_path="/root/path.sql", + language="sql", + raw_code=" \n", + name="foo", + resource_type=NodeType.Model, + ) + self.assert_symmetric(node, node_dict) + self.assertTrue(node.empty) + + self.assert_fails_validation(node_dict, cls=UnparsedRunHook) + self.assert_fails_validation(node_dict, cls=UnparsedMacro) + + +class TestUnparsedRunHook(ContractTestCase): + ContractType = UnparsedRunHook + + def test_ok(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Operation, + "path": "/root/dbt_project.yml", + "original_file_path": "/root/dbt_project.yml", + "package_name": "test", + "language": "sql", + "raw_code": "GRANT select on dbt_postgres", + "index": 4, + } + node = self.ContractType( + package_name="test", + path="/root/dbt_project.yml", + original_file_path="/root/dbt_project.yml", + language="sql", + raw_code="GRANT select on dbt_postgres", + name="foo", + resource_type=NodeType.Operation, + index=4, + ) + self.assert_symmetric(node, node_dict) + self.assert_fails_validation(node_dict, cls=UnparsedNode) + pickle.loads(pickle.dumps(node)) + + def test_bad_type(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Model, # invalid + "path": "/root/dbt_project.yml", + "original_file_path": "/root/dbt_project.yml", + "package_name": "test", + "language": "sql", + "raw_code": "GRANT select on dbt_postgres", + "index": 4, + } + self.assert_fails_validation(node_dict) + + +class TestFreshnessThreshold(ContractTestCase): + ContractType = FreshnessThreshold + + def test_empty(self): + empty = self.ContractType() + self.assert_symmetric(empty, {"error_after": {}, "warn_after": {}}) + self.assertEqual(empty.status(float("Inf")), FreshnessStatus.Pass) + self.assertEqual(empty.status(0), FreshnessStatus.Pass) + + def test_both(self): + threshold = self.ContractType( + warn_after=Time(count=18, period=TimePeriod.hour), + error_after=Time(count=2, period=TimePeriod.day), + ) + dct = { + "error_after": {"count": 2, "period": "day"}, + "warn_after": {"count": 18, "period": "hour"}, + } + self.assert_symmetric(threshold, dct) + + error_seconds = timedelta(days=3).total_seconds() + warn_seconds = timedelta(days=1).total_seconds() + pass_seconds = timedelta(hours=3).total_seconds() + self.assertEqual(threshold.status(error_seconds), FreshnessStatus.Error) + self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) + self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) + pickle.loads(pickle.dumps(threshold)) + + def test_merged(self): + t1 = self.ContractType( + warn_after=Time(count=36, period=TimePeriod.hour), + error_after=Time(count=2, period=TimePeriod.day), + ) + t2 = self.ContractType( + warn_after=Time(count=18, period=TimePeriod.hour), + ) + threshold = self.ContractType( + warn_after=Time(count=18, period=TimePeriod.hour), + error_after=Time(count=None, period=None), + ) + self.assertEqual(threshold, t1.merged(t2)) + + warn_seconds = timedelta(days=1).total_seconds() + pass_seconds = timedelta(hours=3).total_seconds() + self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) + self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) + + +class TestQuoting(ContractTestCase): + ContractType = Quoting + + def test_empty(self): + empty = self.ContractType() + self.assert_symmetric(empty, {}) + + def test_partial(self): + a = self.ContractType(None, True, False) + b = self.ContractType(True, False, None) + self.assert_symmetric(a, {"schema": True, "identifier": False}) + self.assert_symmetric(b, {"database": True, "schema": False}) + + c = a.merged(b) + self.assertEqual(c, self.ContractType(True, False, False)) + self.assert_symmetric(c, {"database": True, "schema": False, "identifier": False}) + pickle.loads(pickle.dumps(c)) + + +class TestUnparsedSourceDefinition(ContractTestCase): + ContractType = UnparsedSourceDefinition + + def test_defaults(self): + minimum = self.ContractType(name="foo") + from_dict = {"name": "foo"} + to_dict = { + "name": "foo", + "description": "", + "freshness": {"error_after": {}, "warn_after": {}}, + "quoting": {}, + "tables": [], + "loader": "", + "meta": {}, + "tags": [], + "config": {}, + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) + + def test_contents(self): + empty = self.ContractType( + name="foo", + description="a description", + quoting=Quoting(database=False), + loader="some_loader", + freshness=FreshnessThreshold(), + tables=[], + meta={}, + ) + dct = { + "name": "foo", + "description": "a description", + "quoting": {"database": False}, + "loader": "some_loader", + "freshness": {"error_after": {}, "warn_after": {}}, + "tables": [], + "meta": {}, + "tags": [], + "config": {}, + } + self.assert_symmetric(empty, dct) + + def test_table_defaults(self): + table_1 = UnparsedSourceTableDefinition(name="table1") + table_2 = UnparsedSourceTableDefinition( + name="table2", + description="table 2", + quoting=Quoting(database=True), + ) + source = self.ContractType(name="foo", tables=[table_1, table_2]) + from_dict = { + "name": "foo", + "tables": [ + {"name": "table1"}, + { + "name": "table2", + "description": "table 2", + "quoting": {"database": True}, + }, + ], + } + to_dict = { + "name": "foo", + "description": "", + "config": {}, + "loader": "", + "freshness": {"error_after": {}, "warn_after": {}}, + "quoting": {}, + "meta": {}, + "tables": [ + { + "name": "table1", + "description": "", + "config": {}, + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "columns": [], + "constraints": [], + "quoting": {}, + "freshness": {"error_after": {}, "warn_after": {}}, + "meta": {}, + "tags": [], + }, + { + "name": "table2", + "description": "table 2", + "config": {}, + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "columns": [], + "constraints": [], + "quoting": {"database": True}, + "freshness": {"error_after": {}, "warn_after": {}}, + "meta": {}, + "tags": [], + }, + ], + "tags": [], + } + self.assert_from_dict(source, from_dict) + self.assert_symmetric(source, to_dict) + pickle.loads(pickle.dumps(source)) + + +class TestUnparsedDocumentationFile(ContractTestCase): + ContractType = UnparsedDocumentationFile + + def test_ok(self): + doc = self.ContractType( + package_name="test", + path="/root/docs", + original_file_path="/root/docs/doc.md", + file_contents="blah blah blah", + ) + doc_dict = { + "package_name": "test", + "path": "/root/docs", + "original_file_path": "/root/docs/doc.md", + "file_contents": "blah blah blah", + } + self.assert_symmetric(doc, doc_dict) + self.assertEqual(doc.resource_type, NodeType.Documentation) + self.assert_fails_validation(doc_dict, UnparsedNode) + pickle.loads(pickle.dumps(doc)) + + def test_extra_field(self): + self.assert_fails_validation({}) + doc_dict = { + "package_name": "test", + "path": "/root/docs", + "original_file_path": "/root/docs/doc.md", + "file_contents": "blah blah blah", + "resource_type": "docs", + } + self.assert_fails_validation(doc_dict) + + +class TestUnparsedNodeUpdate(ContractTestCase): + ContractType = UnparsedNodeUpdate + + def test_defaults(self): + minimum = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + ) + from_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + } + to_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "columns": [], + "description": "", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {}, + "config": {}, + "constraints": [], + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) + + def test_contents(self): + update = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + description="a description", + data_tests=["table_test"], + meta={"key": ["value1", "value2"]}, + columns=[ + UnparsedColumn( + name="x", + description="x description", + meta={"key2": "value3"}, + ), + UnparsedColumn( + name="y", + description="y description", + data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], + meta={}, + tags=["a", "b"], + ), + ], + docs=Docs(show=False), + ) + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "constraints": [], + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + "tags": [], + "constraints": [], + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "tags": ["a", "b"], + "constraints": [], + }, + ], + "docs": {"show": False}, + "config": {}, + } + self.assert_symmetric(update, dct) + pickle.loads(pickle.dumps(update)) + + def test_bad_test_type(self): + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + # column missing a name + { + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + # missing a name + dct = { + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + +class TestUnparsedModelUpdate(ContractTestCase): + ContractType = UnparsedModelUpdate + + def test_defaults(self): + minimum = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + ) + from_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + } + to_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "columns": [], + "description": "", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {}, + "config": {}, + "constraints": [], + "versions": [], + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) + + def test_contents(self): + update = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + description="a description", + data_tests=["table_test"], + meta={"key": ["value1", "value2"]}, + columns=[ + UnparsedColumn( + name="x", + description="x description", + meta={"key2": "value3"}, + ), + UnparsedColumn( + name="y", + description="y description", + data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], + meta={}, + tags=["a", "b"], + ), + ], + docs=Docs(show=False), + versions=[UnparsedVersion(v=2)], + ) + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "constraints": [], + "versions": [ + { + "v": 2, + "description": "", + "columns": [], + "config": {}, + "constraints": [], + "docs": {"show": True}, + } + ], + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + "tags": [], + "constraints": [], + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "tags": ["a", "b"], + "constraints": [], + }, + ], + "docs": {"show": False}, + "config": {}, + } + self.assert_symmetric(update, dct) + pickle.loads(pickle.dumps(update)) + + def test_bad_test_type(self): + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + # column missing a name + { + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + # missing a name + dct = { + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + +class TestUnparsedExposure(ContractTestCase): + ContractType = UnparsedExposure + + def get_ok_dict(self): + return { + "name": "my_exposure", + "type": "dashboard", + "owner": {"name": "example", "email": "name@example.com", "slack": "#channel"}, + "maturity": "medium", + "meta": {"tool": "my_tool"}, + "tags": ["my_department"], + "url": "https://example.com/dashboards/1", + "description": "A exposure", + "config": {}, + "depends_on": [ + 'ref("my_model")', + 'source("raw", "source_table")', + ], + } + + def test_ok(self): + exposure = self.ContractType( + name="my_exposure", + type=ExposureType.Dashboard, + owner=Owner(name="example", email="name@example.com", _extra={"slack": "#channel"}), + maturity=MaturityType.Medium, + url="https://example.com/dashboards/1", + description="A exposure", + config={}, + meta={"tool": "my_tool"}, + tags=["my_department"], + depends_on=['ref("my_model")', 'source("raw", "source_table")'], + ) + dct = self.get_ok_dict() + self.assert_symmetric(exposure, dct) + pickle.loads(pickle.dumps(exposure)) + + def test_ok_exposures(self): + for exposure_allowed in ("dashboard", "notebook", "analysis", "ml", "application"): + tst = self.get_ok_dict() + tst["type"] = exposure_allowed + assert self.ContractType.from_dict(tst).type == exposure_allowed + + def test_bad_exposure(self): + # bad exposure: None isn't allowed + for exposure_not_allowed in (None, "not an exposure"): + tst = self.get_ok_dict() + tst["type"] = exposure_not_allowed + self.assert_fails_validation(tst) + + def test_no_exposure(self): + tst = self.get_ok_dict() + del tst["type"] + self.assert_fails_validation(tst) + + def test_ok_maturities(self): + for maturity_allowed in (None, "low", "medium", "high"): + tst = self.get_ok_dict() + tst["maturity"] = maturity_allowed + assert self.ContractType.from_dict(tst).maturity == maturity_allowed + + tst = self.get_ok_dict() + del tst["maturity"] + assert self.ContractType.from_dict(tst).maturity is None + + def test_bad_maturity(self): + tst = self.get_ok_dict() + tst["maturity"] = "invalid maturity" + self.assert_fails_validation(tst) + + def test_bad_owner_missing_things(self): + tst = self.get_ok_dict() + del tst["owner"]["email"] + del tst["owner"]["name"] + self.assert_fails_validation(tst) + + del tst["owner"] + self.assert_fails_validation(tst) + + def test_bad_tags(self): + tst = self.get_ok_dict() + tst["tags"] = [123] + self.assert_fails_validation(tst) + + +class TestUnparsedMetric(ContractTestCase): + ContractType = UnparsedMetric + + def get_ok_dict(self): + return { + "name": "new_customers", + "label": "New Customers", + "description": "New customers", + "type": "simple", + "type_params": { + "measure": { + "name": "customers", + "filter": "is_new = true", + "join_to_timespine": False, + }, + }, + "config": {}, + "tags": [], + "meta": {"is_okr": True}, + } + + def test_ok(self): + metric = self.ContractType( + name="new_customers", + label="New Customers", + description="New customers", + type="simple", + type_params=UnparsedMetricTypeParams( + measure=UnparsedMetricInputMeasure( + name="customers", + filter="is_new = true", + ) + ), + config={}, + meta={"is_okr": True}, + ) + dct = self.get_ok_dict() + self.assert_symmetric(metric, dct) + pickle.loads(pickle.dumps(metric)) + + def test_bad_metric_no_type_params(self): + tst = self.get_ok_dict() + del tst["type_params"] + self.assert_fails_validation(tst) + + def test_bad_tags(self): + tst = self.get_ok_dict() + tst["tags"] = [123] + self.assert_fails_validation(tst) + + +class TestUnparsedVersion(ContractTestCase): + ContractType = UnparsedVersion + + def get_ok_dict(self): + return { + "v": 2, + "defined_in": "test_defined_in", + "description": "A version", + "config": {}, + "constraints": [], + "docs": {"show": False}, + "data_tests": [], + "columns": [], + } + + def test_ok(self): + version = self.ContractType( + v=2, + defined_in="test_defined_in", + description="A version", + config={}, + constraints=[], + docs=Docs(show=False), + data_tests=[], + columns=[], + ) + dct = self.get_ok_dict() + self.assert_symmetric(version, dct) + pickle.loads(pickle.dumps(version)) + + def test_bad_version_no_v(self): + version = self.get_ok_dict() + del version["v"] + self.assert_fails_validation(version) + + +@pytest.mark.parametrize( + "left,right,expected_lt", + [ + # same types + (2, 12, True), + (12, 2, False), + ("a", "b", True), + ("b", "a", False), + # mismatched types - numeric + (2, 12.0, True), + (12.0, 2, False), + (2, "12", True), + ("12", 2, False), + # mismatched types + (1, "test", True), + ("test", 1, False), + ], +) +def test_unparsed_version_lt(left, right, expected_lt): + assert (UnparsedVersion(left) < UnparsedVersion(right)) == expected_lt def test_column_parse(): diff --git a/tests/unit/test_contracts_project.py b/tests/unit/contracts/test_project.py similarity index 95% rename from tests/unit/test_contracts_project.py rename to tests/unit/contracts/test_project.py index b080326fcd2..37e57a33c12 100644 --- a/tests/unit/test_contracts_project.py +++ b/tests/unit/contracts/test_project.py @@ -1,7 +1,6 @@ from dbt.contracts.project import Project from dbt_common.dataclass_schema import ValidationError - -from .utils import ContractTestCase +from tests.unit.utils import ContractTestCase class TestProject(ContractTestCase): diff --git a/tests/unit/deps/__init__.py b/tests/unit/deps/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_deps.py b/tests/unit/deps/test_deps.py similarity index 100% rename from tests/unit/test_deps.py rename to tests/unit/deps/test_deps.py diff --git a/tests/unit/events/__init__.py b/tests/unit/events/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_proto_events.py b/tests/unit/events/test_types.py similarity index 100% rename from tests/unit/test_proto_events.py rename to tests/unit/events/test_types.py diff --git a/tests/unit/test_graph_selector_parsing.py b/tests/unit/graph/test_cli.py similarity index 100% rename from tests/unit/test_graph_selector_parsing.py rename to tests/unit/graph/test_cli.py diff --git a/tests/unit/graph/test_nodes.py b/tests/unit/graph/test_nodes.py index 897eee5111c..4a62db34b11 100644 --- a/tests/unit/graph/test_nodes.py +++ b/tests/unit/graph/test_nodes.py @@ -9,10 +9,17 @@ EntityType, ) -from dbt.artifacts.resources import Defaults, Dimension, Entity, Measure +from dbt.artifacts.resources import Defaults, Dimension, Entity, Measure, TestMetadata from dbt.artifacts.resources.v1.semantic_model import NodeRelation -from dbt.contracts.graph.nodes import SemanticModel +from dbt.contracts.graph.model_config import TestConfig +from dbt.contracts.graph.nodes import ColumnInfo, SemanticModel from dbt.node_types import NodeType +from dbt_common.contracts.constraints import ( + ColumnLevelConstraint, + ConstraintType, + ModelLevelConstraint, +) +from tests.unit.fixtures import generic_test_node, model_node class TestSemanticModel: @@ -109,3 +116,189 @@ def test_semantic_model_same_contents_different_node_relation( default_semantic_model_copy.node_relation.alias = "test_another_alias" # Relation should not be consided in same_contents assert default_semantic_model.same_contents(default_semantic_model_copy) + + +# Infer primary key +def test_no_primary_key(): + model = model_node() + assert model.infer_primary_key([]) == [] + + +def test_primary_key_model_constraint(): + model = model_node() + model.constraints = [ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk"])] + assertSameContents(model.infer_primary_key([]), ["pk"]) + + model.constraints = [ + ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk1", "pk2"]) + ] + assertSameContents(model.infer_primary_key([]), ["pk1", "pk2"]) + + +def test_primary_key_column_constraint(): + model = model_node() + model.columns = { + "column1": ColumnInfo( + "column1", constraints=[ColumnLevelConstraint(type=ConstraintType.primary_key)] + ), + "column2": ColumnInfo("column2"), + } + assertSameContents(model.infer_primary_key([]), ["column1"]) + + +def test_unique_non_null_single(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column1"}) + test3 = generic_test_node() + test3.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2"}) + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1"]) + + +def test_unique_non_null_multiple(): + model = model_node() + tests = [] + for i in range(2): + for enabled in [True, False]: + test1 = generic_test_node() + test1.test_metadata = TestMetadata( + name="unique", kwargs={"column_name": "column" + str(i) + str(enabled)} + ) + test1.config = TestConfig(enabled=enabled) + test2 = generic_test_node() + test2.test_metadata = TestMetadata( + name="not_null", kwargs={"column_name": "column" + str(i) + str(enabled)} + ) + test2.config = TestConfig(enabled=enabled) + tests.extend([test1, test2]) + + assertSameContents( + model.infer_primary_key(tests), + ["column0True", "column1True", "column0False", "column1False"], + ) + + +def test_enabled_unique_single(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column3"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1"]) + + +def test_enabled_unique_multiple(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) + + +def test_enabled_unique_combo_single(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata( + name="unique_combination_of_columns", + kwargs={"combination_of_columns": ["column1", "column2"]}, + ) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata( + name="unique_combination_of_columns", + kwargs={"combination_of_columns": ["column3", "column4"]}, + ) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) + + +def test_enabled_unique_combo_multiple(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} + ) + test2 = generic_test_node() + test2.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} + ) + + tests = [test1, test2] + assertSameContents( + model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] + ) + + +def test_disabled_unique_single(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column2"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1"]) + + +def test_disabled_unique_multiple(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) + + +def test_disabled_unique_combo_single(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} + ) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata( + name="random", kwargs={"combination_of_columns": ["column3", "column4"]} + ) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) + + +def test_disabled_unique_combo_multiple(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} + ) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} + ) + + tests = [test1, test2] + assertSameContents( + model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] + ) + + +def assertSameContents(list1, list2): + assert sorted(list1) == sorted(list2) diff --git a/tests/unit/graph/test_selector_spec.py b/tests/unit/graph/test_selector_spec.py index 265e387bc0a..451b107d85c 100644 --- a/tests/unit/graph/test_selector_spec.py +++ b/tests/unit/graph/test_selector_spec.py @@ -1,8 +1,17 @@ +import os from unittest.mock import patch import pytest -from dbt.graph.selector_spec import IndirectSelection, SelectionCriteria +from dbt.exceptions import DbtRuntimeError +from dbt.graph.selector_methods import MethodName +from dbt.graph.selector_spec import ( + IndirectSelection, + SelectionCriteria, + SelectionDifference, + SelectionIntersection, + SelectionUnion, +) @pytest.mark.parametrize( @@ -47,3 +56,151 @@ def test_selection_criteria_default_indirect_value(indirect_selection_value, exp assert ( selection_criteria_with_indirect_selection_specified.indirect_selection == "buildable" ) + + +def test_raw_parse_simple(): + raw = "asdf" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == raw + assert not result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_path(): + raw = os.path.join("asdf", "*") + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.Path + assert result.method_arguments == [] + assert result.value == raw + assert not result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_path_modified(): + raw = "@" + os.path.join("asdf", "*") + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.Path + assert result.method_arguments == [] + assert result.value == raw[1:] + assert result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_fqn_parents(): + raw = "+asdf" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "asdf" + assert not result.childrens_parents + assert not result.children + assert result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_fqn_children(): + raw = "asdf+" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "asdf" + assert not result.childrens_parents + assert result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_complex(): + raw = "2+config.arg.secondarg:argument_value+4" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.Config + assert result.method_arguments == ["arg", "secondarg"] + assert result.value == "argument_value" + assert not result.childrens_parents + assert result.children + assert result.parents + assert result.parents_depth == 2 + assert result.children_depth == 4 + + +def test_raw_parse_weird(): + # you can have an empty method name (defaults to FQN/path) and you can have + # an empty value, so you can also have this... + result = SelectionCriteria.from_single_spec("") + assert result.raw == "" + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "" + assert not result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_invalid(): + with pytest.raises(DbtRuntimeError): + SelectionCriteria.from_single_spec("invalid_method:something") + + with pytest.raises(DbtRuntimeError): + SelectionCriteria.from_single_spec("@foo+") + + +def test_intersection(): + fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") + fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") + intersection = SelectionIntersection(components=[fqn_a, fqn_b]) + assert list(intersection) == [fqn_a, fqn_b] + combined = intersection.combine_selections( + [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] + ) + assert combined == {"model_c"} + + +def test_difference(): + fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") + fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") + difference = SelectionDifference(components=[fqn_a, fqn_b]) + assert list(difference) == [fqn_a, fqn_b] + combined = difference.combine_selections( + [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] + ) + assert combined == {"model_a", "model_b"} + + fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") + difference = SelectionDifference(components=[fqn_a, fqn_b, fqn_c]) + assert list(difference) == [fqn_a, fqn_b, fqn_c] + combined = difference.combine_selections( + [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}, {"model_a"}] + ) + assert combined == {"model_b"} + + +def test_union(): + fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") + fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") + fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") + difference = SelectionUnion(components=[fqn_a, fqn_b, fqn_c]) + combined = difference.combine_selections( + [{"model_a", "model_b"}, {"model_b", "model_c"}, {"model_d"}] + ) + assert combined == {"model_a", "model_b", "model_c", "model_d"} diff --git a/tests/unit/parser/__init__.py b/tests/unit/parser/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_docs_blocks.py b/tests/unit/parser/test_docs.py similarity index 99% rename from tests/unit/test_docs_blocks.py rename to tests/unit/parser/test_docs.py index cb1b5e5e3cc..2eb5d12c383 100644 --- a/tests/unit/test_docs_blocks.py +++ b/tests/unit/parser/test_docs.py @@ -9,8 +9,7 @@ from dbt.node_types import NodeType from dbt.parser import docs from dbt.parser.search import FileBlock - -from .utils import config_from_parts_or_dicts +from tests.unit.utils import config_from_parts_or_dicts set_from_args(Namespace(WARN_ERROR=False), None) diff --git a/tests/unit/test_parser.py b/tests/unit/parser/test_parser.py similarity index 99% rename from tests/unit/test_parser.py rename to tests/unit/parser/test_parser.py index c57dd306dcd..42398f48f39 100644 --- a/tests/unit/test_parser.py +++ b/tests/unit/parser/test_parser.py @@ -53,8 +53,12 @@ ) from dbt.parser.search import FileBlock from dbt.parser.sources import SourcePatcher - -from .utils import MockNode, config_from_parts_or_dicts, generate_name_macros, normalize +from tests.unit.utils import ( + MockNode, + config_from_parts_or_dicts, + generate_name_macros, + normalize, +) set_from_args(Namespace(WARN_ERROR=False), None) diff --git a/tests/unit/parser/test_unit_tests.py b/tests/unit/parser/test_unit_tests.py index 39832fa89f9..e8725bed718 100644 --- a/tests/unit/parser/test_unit_tests.py +++ b/tests/unit/parser/test_unit_tests.py @@ -5,7 +5,7 @@ from dbt.contracts.graph.unparsed import UnitTestOutputFixture from dbt.parser import SchemaParser from dbt.parser.unit_tests import UnitTestParser -from tests.unit.test_parser import SchemaParserTest, assertEqualNodes +from tests.unit.parser.test_parser import SchemaParserTest, assertEqualNodes from tests.unit.utils import MockNode UNIT_TEST_MODEL_NOT_FOUND_SOURCE = """ diff --git a/tests/unit/task/__init__.py b/tests/unit/task/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_docs_generate.py b/tests/unit/task/test_docs.py similarity index 100% rename from tests/unit/test_docs_generate.py rename to tests/unit/task/test_docs.py diff --git a/tests/unit/test_contracts_graph_unparsed.py b/tests/unit/test_contracts_graph_unparsed.py deleted file mode 100644 index e1105b80820..00000000000 --- a/tests/unit/test_contracts_graph_unparsed.py +++ /dev/null @@ -1,986 +0,0 @@ -import pickle -from datetime import timedelta - -import pytest - -from dbt.artifacts.resources import ( - ExposureType, - FreshnessThreshold, - MaturityType, - Owner, - Quoting, - Time, -) -from dbt.artifacts.resources.types import TimePeriod -from dbt.artifacts.schemas.results import FreshnessStatus -from dbt.contracts.graph.unparsed import ( - Docs, - UnparsedColumn, - UnparsedDocumentationFile, - UnparsedExposure, - UnparsedMacro, - UnparsedMetric, - UnparsedMetricInputMeasure, - UnparsedMetricTypeParams, - UnparsedModelUpdate, - UnparsedNode, - UnparsedNodeUpdate, - UnparsedRunHook, - UnparsedSourceDefinition, - UnparsedSourceTableDefinition, - UnparsedVersion, -) -from dbt.node_types import NodeType - -from .utils import ContractTestCase - - -class TestUnparsedMacro(ContractTestCase): - ContractType = UnparsedMacro - - def test_ok(self): - macro_dict = { - "path": "/root/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", - "resource_type": "macro", - } - macro = self.ContractType( - path="/root/path.sql", - original_file_path="/root/path.sql", - package_name="test", - language="sql", - raw_code="{% macro foo() %}select 1 as id{% endmacro %}", - resource_type=NodeType.Macro, - ) - self.assert_symmetric(macro, macro_dict) - pickle.loads(pickle.dumps(macro)) - - def test_invalid_missing_field(self): - macro_dict = { - "path": "/root/path.sql", - "original_file_path": "/root/path.sql", - # 'package_name': 'test', - "language": "sql", - "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", - "resource_type": "macro", - } - self.assert_fails_validation(macro_dict) - - def test_invalid_extra_field(self): - macro_dict = { - "path": "/root/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", - "extra": "extra", - "resource_type": "macro", - } - self.assert_fails_validation(macro_dict) - - -class TestUnparsedNode(ContractTestCase): - ContractType = UnparsedNode - - def test_ok(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Model, - "path": "/root/x/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": 'select * from {{ ref("thing") }}', - } - node = self.ContractType( - package_name="test", - path="/root/x/path.sql", - original_file_path="/root/path.sql", - language="sql", - raw_code='select * from {{ ref("thing") }}', - name="foo", - resource_type=NodeType.Model, - ) - self.assert_symmetric(node, node_dict) - self.assertFalse(node.empty) - - self.assert_fails_validation(node_dict, cls=UnparsedRunHook) - self.assert_fails_validation(node_dict, cls=UnparsedMacro) - pickle.loads(pickle.dumps(node)) - - def test_empty(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Model, - "path": "/root/x/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": " \n", - } - node = UnparsedNode( - package_name="test", - path="/root/x/path.sql", - original_file_path="/root/path.sql", - language="sql", - raw_code=" \n", - name="foo", - resource_type=NodeType.Model, - ) - self.assert_symmetric(node, node_dict) - self.assertTrue(node.empty) - - self.assert_fails_validation(node_dict, cls=UnparsedRunHook) - self.assert_fails_validation(node_dict, cls=UnparsedMacro) - - -class TestUnparsedRunHook(ContractTestCase): - ContractType = UnparsedRunHook - - def test_ok(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Operation, - "path": "/root/dbt_project.yml", - "original_file_path": "/root/dbt_project.yml", - "package_name": "test", - "language": "sql", - "raw_code": "GRANT select on dbt_postgres", - "index": 4, - } - node = self.ContractType( - package_name="test", - path="/root/dbt_project.yml", - original_file_path="/root/dbt_project.yml", - language="sql", - raw_code="GRANT select on dbt_postgres", - name="foo", - resource_type=NodeType.Operation, - index=4, - ) - self.assert_symmetric(node, node_dict) - self.assert_fails_validation(node_dict, cls=UnparsedNode) - pickle.loads(pickle.dumps(node)) - - def test_bad_type(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Model, # invalid - "path": "/root/dbt_project.yml", - "original_file_path": "/root/dbt_project.yml", - "package_name": "test", - "language": "sql", - "raw_code": "GRANT select on dbt_postgres", - "index": 4, - } - self.assert_fails_validation(node_dict) - - -class TestFreshnessThreshold(ContractTestCase): - ContractType = FreshnessThreshold - - def test_empty(self): - empty = self.ContractType() - self.assert_symmetric(empty, {"error_after": {}, "warn_after": {}}) - self.assertEqual(empty.status(float("Inf")), FreshnessStatus.Pass) - self.assertEqual(empty.status(0), FreshnessStatus.Pass) - - def test_both(self): - threshold = self.ContractType( - warn_after=Time(count=18, period=TimePeriod.hour), - error_after=Time(count=2, period=TimePeriod.day), - ) - dct = { - "error_after": {"count": 2, "period": "day"}, - "warn_after": {"count": 18, "period": "hour"}, - } - self.assert_symmetric(threshold, dct) - - error_seconds = timedelta(days=3).total_seconds() - warn_seconds = timedelta(days=1).total_seconds() - pass_seconds = timedelta(hours=3).total_seconds() - self.assertEqual(threshold.status(error_seconds), FreshnessStatus.Error) - self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) - self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) - pickle.loads(pickle.dumps(threshold)) - - def test_merged(self): - t1 = self.ContractType( - warn_after=Time(count=36, period=TimePeriod.hour), - error_after=Time(count=2, period=TimePeriod.day), - ) - t2 = self.ContractType( - warn_after=Time(count=18, period=TimePeriod.hour), - ) - threshold = self.ContractType( - warn_after=Time(count=18, period=TimePeriod.hour), - error_after=Time(count=None, period=None), - ) - self.assertEqual(threshold, t1.merged(t2)) - - warn_seconds = timedelta(days=1).total_seconds() - pass_seconds = timedelta(hours=3).total_seconds() - self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) - self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) - - -class TestQuoting(ContractTestCase): - ContractType = Quoting - - def test_empty(self): - empty = self.ContractType() - self.assert_symmetric(empty, {}) - - def test_partial(self): - a = self.ContractType(None, True, False) - b = self.ContractType(True, False, None) - self.assert_symmetric(a, {"schema": True, "identifier": False}) - self.assert_symmetric(b, {"database": True, "schema": False}) - - c = a.merged(b) - self.assertEqual(c, self.ContractType(True, False, False)) - self.assert_symmetric(c, {"database": True, "schema": False, "identifier": False}) - pickle.loads(pickle.dumps(c)) - - -class TestUnparsedSourceDefinition(ContractTestCase): - ContractType = UnparsedSourceDefinition - - def test_defaults(self): - minimum = self.ContractType(name="foo") - from_dict = {"name": "foo"} - to_dict = { - "name": "foo", - "description": "", - "freshness": {"error_after": {}, "warn_after": {}}, - "quoting": {}, - "tables": [], - "loader": "", - "meta": {}, - "tags": [], - "config": {}, - } - self.assert_from_dict(minimum, from_dict) - self.assert_to_dict(minimum, to_dict) - - def test_contents(self): - empty = self.ContractType( - name="foo", - description="a description", - quoting=Quoting(database=False), - loader="some_loader", - freshness=FreshnessThreshold(), - tables=[], - meta={}, - ) - dct = { - "name": "foo", - "description": "a description", - "quoting": {"database": False}, - "loader": "some_loader", - "freshness": {"error_after": {}, "warn_after": {}}, - "tables": [], - "meta": {}, - "tags": [], - "config": {}, - } - self.assert_symmetric(empty, dct) - - def test_table_defaults(self): - table_1 = UnparsedSourceTableDefinition(name="table1") - table_2 = UnparsedSourceTableDefinition( - name="table2", - description="table 2", - quoting=Quoting(database=True), - ) - source = self.ContractType(name="foo", tables=[table_1, table_2]) - from_dict = { - "name": "foo", - "tables": [ - {"name": "table1"}, - { - "name": "table2", - "description": "table 2", - "quoting": {"database": True}, - }, - ], - } - to_dict = { - "name": "foo", - "description": "", - "config": {}, - "loader": "", - "freshness": {"error_after": {}, "warn_after": {}}, - "quoting": {}, - "meta": {}, - "tables": [ - { - "name": "table1", - "description": "", - "config": {}, - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "columns": [], - "constraints": [], - "quoting": {}, - "freshness": {"error_after": {}, "warn_after": {}}, - "meta": {}, - "tags": [], - }, - { - "name": "table2", - "description": "table 2", - "config": {}, - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "columns": [], - "constraints": [], - "quoting": {"database": True}, - "freshness": {"error_after": {}, "warn_after": {}}, - "meta": {}, - "tags": [], - }, - ], - "tags": [], - } - self.assert_from_dict(source, from_dict) - self.assert_symmetric(source, to_dict) - pickle.loads(pickle.dumps(source)) - - -class TestUnparsedDocumentationFile(ContractTestCase): - ContractType = UnparsedDocumentationFile - - def test_ok(self): - doc = self.ContractType( - package_name="test", - path="/root/docs", - original_file_path="/root/docs/doc.md", - file_contents="blah blah blah", - ) - doc_dict = { - "package_name": "test", - "path": "/root/docs", - "original_file_path": "/root/docs/doc.md", - "file_contents": "blah blah blah", - } - self.assert_symmetric(doc, doc_dict) - self.assertEqual(doc.resource_type, NodeType.Documentation) - self.assert_fails_validation(doc_dict, UnparsedNode) - pickle.loads(pickle.dumps(doc)) - - def test_extra_field(self): - self.assert_fails_validation({}) - doc_dict = { - "package_name": "test", - "path": "/root/docs", - "original_file_path": "/root/docs/doc.md", - "file_contents": "blah blah blah", - "resource_type": "docs", - } - self.assert_fails_validation(doc_dict) - - -class TestUnparsedNodeUpdate(ContractTestCase): - ContractType = UnparsedNodeUpdate - - def test_defaults(self): - minimum = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - ) - from_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - } - to_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "columns": [], - "description": "", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {}, - "config": {}, - "constraints": [], - } - self.assert_from_dict(minimum, from_dict) - self.assert_to_dict(minimum, to_dict) - - def test_contents(self): - update = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - description="a description", - data_tests=["table_test"], - meta={"key": ["value1", "value2"]}, - columns=[ - UnparsedColumn( - name="x", - description="x description", - meta={"key2": "value3"}, - ), - UnparsedColumn( - name="y", - description="y description", - data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], - meta={}, - tags=["a", "b"], - ), - ], - docs=Docs(show=False), - ) - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "constraints": [], - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - "tags": [], - "constraints": [], - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "tags": ["a", "b"], - "constraints": [], - }, - ], - "docs": {"show": False}, - "config": {}, - } - self.assert_symmetric(update, dct) - pickle.loads(pickle.dumps(update)) - - def test_bad_test_type(self): - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - # column missing a name - { - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - # missing a name - dct = { - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - -class TestUnparsedModelUpdate(ContractTestCase): - ContractType = UnparsedModelUpdate - - def test_defaults(self): - minimum = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - ) - from_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - } - to_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "columns": [], - "description": "", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {}, - "config": {}, - "constraints": [], - "versions": [], - } - self.assert_from_dict(minimum, from_dict) - self.assert_to_dict(minimum, to_dict) - - def test_contents(self): - update = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - description="a description", - data_tests=["table_test"], - meta={"key": ["value1", "value2"]}, - columns=[ - UnparsedColumn( - name="x", - description="x description", - meta={"key2": "value3"}, - ), - UnparsedColumn( - name="y", - description="y description", - data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], - meta={}, - tags=["a", "b"], - ), - ], - docs=Docs(show=False), - versions=[UnparsedVersion(v=2)], - ) - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "constraints": [], - "versions": [ - { - "v": 2, - "description": "", - "columns": [], - "config": {}, - "constraints": [], - "docs": {"show": True}, - } - ], - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - "tags": [], - "constraints": [], - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "tags": ["a", "b"], - "constraints": [], - }, - ], - "docs": {"show": False}, - "config": {}, - } - self.assert_symmetric(update, dct) - pickle.loads(pickle.dumps(update)) - - def test_bad_test_type(self): - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - # column missing a name - { - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - # missing a name - dct = { - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - -class TestUnparsedExposure(ContractTestCase): - ContractType = UnparsedExposure - - def get_ok_dict(self): - return { - "name": "my_exposure", - "type": "dashboard", - "owner": {"name": "example", "email": "name@example.com", "slack": "#channel"}, - "maturity": "medium", - "meta": {"tool": "my_tool"}, - "tags": ["my_department"], - "url": "https://example.com/dashboards/1", - "description": "A exposure", - "config": {}, - "depends_on": [ - 'ref("my_model")', - 'source("raw", "source_table")', - ], - } - - def test_ok(self): - exposure = self.ContractType( - name="my_exposure", - type=ExposureType.Dashboard, - owner=Owner(name="example", email="name@example.com", _extra={"slack": "#channel"}), - maturity=MaturityType.Medium, - url="https://example.com/dashboards/1", - description="A exposure", - config={}, - meta={"tool": "my_tool"}, - tags=["my_department"], - depends_on=['ref("my_model")', 'source("raw", "source_table")'], - ) - dct = self.get_ok_dict() - self.assert_symmetric(exposure, dct) - pickle.loads(pickle.dumps(exposure)) - - def test_ok_exposures(self): - for exposure_allowed in ("dashboard", "notebook", "analysis", "ml", "application"): - tst = self.get_ok_dict() - tst["type"] = exposure_allowed - assert self.ContractType.from_dict(tst).type == exposure_allowed - - def test_bad_exposure(self): - # bad exposure: None isn't allowed - for exposure_not_allowed in (None, "not an exposure"): - tst = self.get_ok_dict() - tst["type"] = exposure_not_allowed - self.assert_fails_validation(tst) - - def test_no_exposure(self): - tst = self.get_ok_dict() - del tst["type"] - self.assert_fails_validation(tst) - - def test_ok_maturities(self): - for maturity_allowed in (None, "low", "medium", "high"): - tst = self.get_ok_dict() - tst["maturity"] = maturity_allowed - assert self.ContractType.from_dict(tst).maturity == maturity_allowed - - tst = self.get_ok_dict() - del tst["maturity"] - assert self.ContractType.from_dict(tst).maturity is None - - def test_bad_maturity(self): - tst = self.get_ok_dict() - tst["maturity"] = "invalid maturity" - self.assert_fails_validation(tst) - - def test_bad_owner_missing_things(self): - tst = self.get_ok_dict() - del tst["owner"]["email"] - del tst["owner"]["name"] - self.assert_fails_validation(tst) - - del tst["owner"] - self.assert_fails_validation(tst) - - def test_bad_tags(self): - tst = self.get_ok_dict() - tst["tags"] = [123] - self.assert_fails_validation(tst) - - -class TestUnparsedMetric(ContractTestCase): - ContractType = UnparsedMetric - - def get_ok_dict(self): - return { - "name": "new_customers", - "label": "New Customers", - "description": "New customers", - "type": "simple", - "type_params": { - "measure": { - "name": "customers", - "filter": "is_new = true", - "join_to_timespine": False, - }, - }, - "config": {}, - "tags": [], - "meta": {"is_okr": True}, - } - - def test_ok(self): - metric = self.ContractType( - name="new_customers", - label="New Customers", - description="New customers", - type="simple", - type_params=UnparsedMetricTypeParams( - measure=UnparsedMetricInputMeasure( - name="customers", - filter="is_new = true", - ) - ), - config={}, - meta={"is_okr": True}, - ) - dct = self.get_ok_dict() - self.assert_symmetric(metric, dct) - pickle.loads(pickle.dumps(metric)) - - def test_bad_metric_no_type_params(self): - tst = self.get_ok_dict() - del tst["type_params"] - self.assert_fails_validation(tst) - - def test_bad_tags(self): - tst = self.get_ok_dict() - tst["tags"] = [123] - self.assert_fails_validation(tst) - - -class TestUnparsedVersion(ContractTestCase): - ContractType = UnparsedVersion - - def get_ok_dict(self): - return { - "v": 2, - "defined_in": "test_defined_in", - "description": "A version", - "config": {}, - "constraints": [], - "docs": {"show": False}, - "data_tests": [], - "columns": [], - } - - def test_ok(self): - version = self.ContractType( - v=2, - defined_in="test_defined_in", - description="A version", - config={}, - constraints=[], - docs=Docs(show=False), - data_tests=[], - columns=[], - ) - dct = self.get_ok_dict() - self.assert_symmetric(version, dct) - pickle.loads(pickle.dumps(version)) - - def test_bad_version_no_v(self): - version = self.get_ok_dict() - del version["v"] - self.assert_fails_validation(version) - - -@pytest.mark.parametrize( - "left,right,expected_lt", - [ - # same types - (2, 12, True), - (12, 2, False), - ("a", "b", True), - ("b", "a", False), - # mismatched types - numeric - (2, 12.0, True), - (12.0, 2, False), - (2, "12", True), - ("12", 2, False), - # mismatched types - (1, "test", True), - ("test", 1, False), - ], -) -def test_unparsed_version_lt(left, right, expected_lt): - assert (UnparsedVersion(left) < UnparsedVersion(right)) == expected_lt diff --git a/tests/unit/test_graph_selector_spec.py b/tests/unit/test_graph_selector_spec.py deleted file mode 100644 index 6164f113158..00000000000 --- a/tests/unit/test_graph_selector_spec.py +++ /dev/null @@ -1,160 +0,0 @@ -import os - -import pytest - -from dbt.exceptions import DbtRuntimeError -from dbt.graph.selector_methods import MethodName -from dbt.graph.selector_spec import ( - SelectionCriteria, - SelectionDifference, - SelectionIntersection, - SelectionUnion, -) - - -def test_raw_parse_simple(): - raw = "asdf" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == raw - assert not result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_path(): - raw = os.path.join("asdf", "*") - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.Path - assert result.method_arguments == [] - assert result.value == raw - assert not result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_path_modified(): - raw = "@" + os.path.join("asdf", "*") - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.Path - assert result.method_arguments == [] - assert result.value == raw[1:] - assert result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_fqn_parents(): - raw = "+asdf" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == "asdf" - assert not result.childrens_parents - assert not result.children - assert result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_fqn_children(): - raw = "asdf+" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == "asdf" - assert not result.childrens_parents - assert result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_complex(): - raw = "2+config.arg.secondarg:argument_value+4" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.Config - assert result.method_arguments == ["arg", "secondarg"] - assert result.value == "argument_value" - assert not result.childrens_parents - assert result.children - assert result.parents - assert result.parents_depth == 2 - assert result.children_depth == 4 - - -def test_raw_parse_weird(): - # you can have an empty method name (defaults to FQN/path) and you can have - # an empty value, so you can also have this... - result = SelectionCriteria.from_single_spec("") - assert result.raw == "" - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == "" - assert not result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_invalid(): - with pytest.raises(DbtRuntimeError): - SelectionCriteria.from_single_spec("invalid_method:something") - - with pytest.raises(DbtRuntimeError): - SelectionCriteria.from_single_spec("@foo+") - - -def test_intersection(): - fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") - fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") - intersection = SelectionIntersection(components=[fqn_a, fqn_b]) - assert list(intersection) == [fqn_a, fqn_b] - combined = intersection.combine_selections( - [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] - ) - assert combined == {"model_c"} - - -def test_difference(): - fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") - fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") - difference = SelectionDifference(components=[fqn_a, fqn_b]) - assert list(difference) == [fqn_a, fqn_b] - combined = difference.combine_selections( - [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] - ) - assert combined == {"model_a", "model_b"} - - fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") - difference = SelectionDifference(components=[fqn_a, fqn_b, fqn_c]) - assert list(difference) == [fqn_a, fqn_b, fqn_c] - combined = difference.combine_selections( - [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}, {"model_a"}] - ) - assert combined == {"model_b"} - - -def test_union(): - fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") - fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") - fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") - difference = SelectionUnion(components=[fqn_a, fqn_b, fqn_c]) - combined = difference.combine_selections( - [{"model_a", "model_b"}, {"model_b", "model_c"}, {"model_d"}] - ) - assert combined == {"model_a", "model_b", "model_c", "model_d"} diff --git a/tests/unit/test_infer_primary_key.py b/tests/unit/test_infer_primary_key.py deleted file mode 100644 index 8500c8ea7fe..00000000000 --- a/tests/unit/test_infer_primary_key.py +++ /dev/null @@ -1,195 +0,0 @@ -from dbt.artifacts.resources import TestMetadata -from dbt.contracts.graph.model_config import TestConfig -from dbt.contracts.graph.nodes import ColumnInfo -from dbt_common.contracts.constraints import ( - ColumnLevelConstraint, - ConstraintType, - ModelLevelConstraint, -) - -from .fixtures import generic_test_node, model_node - - -def test_no_primary_key(): - model = model_node() - assert model.infer_primary_key([]) == [] - - -def test_primary_key_model_constraint(): - model = model_node() - model.constraints = [ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk"])] - assertSameContents(model.infer_primary_key([]), ["pk"]) - - model.constraints = [ - ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk1", "pk2"]) - ] - assertSameContents(model.infer_primary_key([]), ["pk1", "pk2"]) - - -def test_primary_key_column_constraint(): - model = model_node() - model.columns = { - "column1": ColumnInfo( - "column1", constraints=[ColumnLevelConstraint(type=ConstraintType.primary_key)] - ), - "column2": ColumnInfo("column2"), - } - assertSameContents(model.infer_primary_key([]), ["column1"]) - - -def test_unique_non_null_single(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column1"}) - test3 = generic_test_node() - test3.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2"}) - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1"]) - - -def test_unique_non_null_multiple(): - model = model_node() - tests = [] - for i in range(2): - for enabled in [True, False]: - test1 = generic_test_node() - test1.test_metadata = TestMetadata( - name="unique", kwargs={"column_name": "column" + str(i) + str(enabled)} - ) - test1.config = TestConfig(enabled=enabled) - test2 = generic_test_node() - test2.test_metadata = TestMetadata( - name="not_null", kwargs={"column_name": "column" + str(i) + str(enabled)} - ) - test2.config = TestConfig(enabled=enabled) - tests.extend([test1, test2]) - - assertSameContents( - model.infer_primary_key(tests), - ["column0True", "column1True", "column0False", "column1False"], - ) - - -def test_enabled_unique_single(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column3"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1"]) - - -def test_enabled_unique_multiple(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) - - -def test_enabled_unique_combo_single(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata( - name="unique_combination_of_columns", - kwargs={"combination_of_columns": ["column1", "column2"]}, - ) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata( - name="unique_combination_of_columns", - kwargs={"combination_of_columns": ["column3", "column4"]}, - ) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) - - -def test_enabled_unique_combo_multiple(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} - ) - test2 = generic_test_node() - test2.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} - ) - - tests = [test1, test2] - assertSameContents( - model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] - ) - - -def test_disabled_unique_single(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column2"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1"]) - - -def test_disabled_unique_multiple(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) - - -def test_disabled_unique_combo_single(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} - ) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata( - name="random", kwargs={"combination_of_columns": ["column3", "column4"]} - ) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) - - -def test_disabled_unique_combo_multiple(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} - ) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} - ) - - tests = [test1, test2] - assertSameContents( - model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] - ) - - -def assertSameContents(list1, list2): - assert sorted(list1) == sorted(list2) diff --git a/tests/unit/test_inject_ctes.py b/tests/unit/test_inject_ctes.py deleted file mode 100644 index e6efdd4f1a6..00000000000 --- a/tests/unit/test_inject_ctes.py +++ /dev/null @@ -1,198 +0,0 @@ -import re - -from dbt.compilation import inject_ctes_into_sql -from dbt.contracts.graph.nodes import InjectedCTE - - -def norm_whitespace(string): - _RE_COMBINE_WHITESPACE = re.compile(r"\s+") - string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() - return string - - -def test_inject_ctes_simple1(): - starting_sql = "select * from __dbt__cte__base" - ctes = [ - InjectedCTE( - id="model.test.base", - sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", - ) - ] - expected_sql = """with __dbt__cte__base as ( - select * from test16873767336887004702_test_ephemeral.seed - ) select * from __dbt__cte__base""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_simple2(): - starting_sql = "select * from __dbt__cte__ephemeral_level_two" - ctes = [ - InjectedCTE( - id="model.test.ephemeral_level_two", - sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873757769710148165_test_ephemeral"."source_table"\n)', - ) - ] - expected_sql = """with __dbt__cte__ephemeral_level_two as ( - select * from "dbt"."test16873757769710148165_test_ephemeral"."source_table" - ) select * from __dbt__cte__ephemeral_level_two""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_multiple_ctes(): - - starting_sql = "select * from __dbt__cte__ephemeral" - ctes = [ - InjectedCTE( - id="model.test.ephemeral_level_two", - sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873735573223965828_test_ephemeral"."source_table"\n)', - ), - InjectedCTE( - id="model.test.ephemeral", - sql=" __dbt__cte__ephemeral as (\n\nselect * from __dbt__cte__ephemeral_level_two\n)", - ), - ] - expected_sql = """with __dbt__cte__ephemeral_level_two as ( - select * from "dbt"."test16873735573223965828_test_ephemeral"."source_table" - ), __dbt__cte__ephemeral as ( - select * from __dbt__cte__ephemeral_level_two - ) select * from __dbt__cte__ephemeral""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_multiple_ctes_more_complex(): - starting_sql = """select * from __dbt__cte__female_only - union all - select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" - ctes = [ - InjectedCTE( - id="model.test.base", - sql=" __dbt__cte__base as (\n\n\nselect * from test16873757723266827902_test_ephemeral.seed\n)", - ), - InjectedCTE( - id="model.test.base_copy", - sql=" __dbt__cte__base_copy as (\n\n\nselect * from __dbt__cte__base\n)", - ), - InjectedCTE( - id="model.test.female_only", - sql=" __dbt__cte__female_only as (\n\n\nselect * from __dbt__cte__base_copy where gender = 'Female'\n)", - ), - ] - expected_sql = """with __dbt__cte__base as ( - select * from test16873757723266827902_test_ephemeral.seed - ), __dbt__cte__base_copy as ( - select * from __dbt__cte__base - ), __dbt__cte__female_only as ( - select * from __dbt__cte__base_copy where gender = 'Female' - ) select * from __dbt__cte__female_only - union all - select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_starting_with1(): - starting_sql = """ - with internal_cte as (select * from sessions) - select * from internal_cte - """ - ctes = [ - InjectedCTE( - id="cte_id_1", - sql="__dbt__cte__ephemeral as (select * from table)", - ), - InjectedCTE( - id="cte_id_2", - sql="__dbt__cte__events as (select id, type from events)", - ), - ] - expected_sql = """with __dbt__cte__ephemeral as (select * from table), - __dbt__cte__events as (select id, type from events), - internal_cte as (select * from sessions) - select * from internal_cte""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_starting_with2(): - starting_sql = """with my_other_cool_cte as ( - select id, name from __dbt__cte__ephemeral - where id > 1000 - ) - select name, id from my_other_cool_cte""" - ctes = [ - InjectedCTE( - id="model.singular_tests_ephemeral.ephemeral", - sql=' __dbt__cte__ephemeral as (\n\n\nwith my_cool_cte as (\n select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base"\n)\nselect id, name from my_cool_cte where id is not null\n)', - ) - ] - expected_sql = """with __dbt__cte__ephemeral as ( - with my_cool_cte as ( - select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base" - ) - select id, name from my_cool_cte where id is not null - ), my_other_cool_cte as ( - select id, name from __dbt__cte__ephemeral - where id > 1000 - ) - select name, id from my_other_cool_cte""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_comment_with(): - # Test injection with a comment containing "with" - starting_sql = """ - --- This is sql with a comment - select * from __dbt__cte__base - """ - ctes = [ - InjectedCTE( - id="model.test.base", - sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", - ) - ] - expected_sql = """with __dbt__cte__base as ( - select * from test16873767336887004702_test_ephemeral.seed - ) --- This is sql with a comment - select * from __dbt__cte__base""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_with_recursive(): - # Test injection with "recursive" keyword - starting_sql = """ - with recursive t(n) as ( - select * from __dbt__cte__first_ephemeral_model - union all - select n+1 from t where n < 100 - ) - select sum(n) from t - """ - ctes = [ - InjectedCTE( - id="model.test.first_ephemeral_model", - sql=" __dbt__cte__first_ephemeral_model as (\n\nselect 1 as fun\n)", - ) - ] - expected_sql = """with recursive __dbt__cte__first_ephemeral_model as ( - select 1 as fun - ), t(n) as ( - select * from __dbt__cte__first_ephemeral_model - union all - select n+1 from t where n < 100 - ) - select sum(n) from t - """ - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) diff --git a/tests/unit/test_deprecations.py b/tests/unit/test_internal_deprecations.py similarity index 100% rename from tests/unit/test_deprecations.py rename to tests/unit/test_internal_deprecations.py diff --git a/tests/unit/test_manifest_selectors.py b/tests/unit/test_manifest_selectors.py deleted file mode 100644 index 36ab31a5874..00000000000 --- a/tests/unit/test_manifest_selectors.py +++ /dev/null @@ -1,201 +0,0 @@ -import textwrap -import unittest -from collections import OrderedDict - -import yaml - -from dbt.config.selectors import SelectorDict -from dbt.exceptions import DbtSelectorsError - - -def get_selector_dict(txt: str) -> OrderedDict: - txt = textwrap.dedent(txt) - dct = OrderedDict(yaml.safe_load(txt)) - return dct - - -class SelectorUnitTest(unittest.TestCase): - def test_compare_cli_non_cli(self): - dct = get_selector_dict( - """\ - selectors: - - name: nightly_diet_snowplow - description: "This uses more CLI-style syntax" - definition: - union: - - intersection: - - '@source:snowplow' - - 'tag:nightly' - - 'models/export' - - exclude: - - intersection: - - 'package:snowplow' - - 'config.materialized:incremental' - - export_performance_timing - - name: nightly_diet_snowplow_full - description: "This is a fuller YAML specification" - definition: - union: - - intersection: - - method: source - value: snowplow - childrens_parents: true - - method: tag - value: nightly - - method: path - value: models/export - - exclude: - - intersection: - - method: package - value: snowplow - - method: config.materialized - value: incremental - - method: fqn - value: export_performance_timing - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - with_strings = sel_dict["nightly_diet_snowplow"]["definition"] - no_strings = sel_dict["nightly_diet_snowplow_full"]["definition"] - self.assertEqual(with_strings, no_strings) - - def test_single_string_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: nightly_selector - definition: - 'tag:nightly' - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "tag", "value": "nightly"} - definition = sel_dict["nightly_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_single_key_value_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: nightly_selector - definition: - tag: nightly - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "tag", "value": "nightly"} - definition = sel_dict["nightly_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_parent_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: kpi_nightly_selector - definition: - '+exposure:kpi_nightly' - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "exposure", "value": "kpi_nightly", "parents": True} - definition = sel_dict["kpi_nightly_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_plus_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: my_model_children_selector - definition: - 'my_model+2' - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "fqn", "value": "my_model", "children": True, "children_depth": "2"} - definition = sel_dict["my_model_children_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_selector_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: default - definition: - union: - - intersection: - - tag: foo - - tag: bar - - name: inherited - definition: - method: selector - value: default - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - definition = sel_dict["default"]["definition"] - expected = sel_dict["inherited"]["definition"] - self.assertEqual(expected, definition) - - def test_selector_definition_with_exclusion(self): - dct = get_selector_dict( - """\ - selectors: - - name: default - definition: - union: - - intersection: - - tag: foo - - tag: bar - - name: inherited - definition: - union: - - method: selector - value: default - - exclude: - - tag: bar - - name: comparison - definition: - union: - - union: - - intersection: - - tag: foo - - tag: bar - - exclude: - - tag: bar - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list((dct["selectors"])) - assert sel_dict - definition = sel_dict["inherited"]["definition"] - expected = sel_dict["comparison"]["definition"] - self.assertEqual(expected, definition) - - def test_missing_selector(self): - dct = get_selector_dict( - """\ - selectors: - - name: inherited - definition: - method: selector - value: default - """ - ) - with self.assertRaises(DbtSelectorsError) as err: - SelectorDict.parse_from_selectors_list((dct["selectors"])) - - self.assertEqual( - "Existing selector definition for default not found.", str(err.exception.msg) - ) diff --git a/tests/unit/test_semver.py b/tests/unit/test_semver.py deleted file mode 100644 index ae48e592fc0..00000000000 --- a/tests/unit/test_semver.py +++ /dev/null @@ -1,298 +0,0 @@ -import itertools -import unittest -from typing import List - -from dbt_common.exceptions import VersionsNotCompatibleError -from dbt_common.semver import ( - UnboundedVersionSpecifier, - VersionRange, - VersionSpecifier, - filter_installable, - reduce_versions, - resolve_to_specific_version, - versions_compatible, -) - - -def semver_regex_versioning(versions: List[str]) -> bool: - for version_string in versions: - try: - VersionSpecifier.from_version_string(version_string) - except Exception: - return False - return True - - -def create_range(start_version_string, end_version_string): - start = UnboundedVersionSpecifier() - end = UnboundedVersionSpecifier() - - if start_version_string is not None: - start = VersionSpecifier.from_version_string(start_version_string) - - if end_version_string is not None: - end = VersionSpecifier.from_version_string(end_version_string) - - return VersionRange(start=start, end=end) - - -class TestSemver(unittest.TestCase): - def assertVersionSetResult(self, inputs, output_range): - expected = create_range(*output_range) - - for permutation in itertools.permutations(inputs): - self.assertEqual(reduce_versions(*permutation), expected) - - def assertInvalidVersionSet(self, inputs): - for permutation in itertools.permutations(inputs): - with self.assertRaises(VersionsNotCompatibleError): - reduce_versions(*permutation) - - def test__versions_compatible(self): - self.assertTrue(versions_compatible("0.0.1", "0.0.1")) - self.assertFalse(versions_compatible("0.0.1", "0.0.2")) - self.assertTrue(versions_compatible(">0.0.1", "0.0.2")) - self.assertFalse(versions_compatible("0.4.5a1", "0.4.5a2")) - - def test__semver_regex_versions(self): - self.assertTrue( - semver_regex_versioning( - [ - "0.0.4", - "1.2.3", - "10.20.30", - "1.1.2-prerelease+meta", - "1.1.2+meta", - "1.1.2+meta-valid", - "1.0.0-alpha", - "1.0.0-beta", - "1.0.0-alpha.beta", - "1.0.0-alpha.beta.1", - "1.0.0-alpha.1", - "1.0.0-alpha0.valid", - "1.0.0-alpha.0valid", - "1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay", - "1.0.0-rc.1+build.1", - "2.0.0-rc.1+build.123", - "1.2.3-beta", - "10.2.3-DEV-SNAPSHOT", - "1.2.3-SNAPSHOT-123", - "1.0.0", - "2.0.0", - "1.1.7", - "2.0.0+build.1848", - "2.0.1-alpha.1227", - "1.0.0-alpha+beta", - "1.2.3----RC-SNAPSHOT.12.9.1--.12+788", - "1.2.3----R-S.12.9.1--.12+meta", - "1.2.3----RC-SNAPSHOT.12.9.1--.12", - "1.0.0+0.build.1-rc.10000aaa-kk-0.1", - "99999999999999999999999.999999999999999999.99999999999999999", - "1.0.0-0A.is.legal", - ] - ) - ) - - self.assertFalse( - semver_regex_versioning( - [ - "1", - "1.2", - "1.2.3-0123", - "1.2.3-0123.0123", - "1.1.2+.123", - "+invalid", - "-invalid", - "-invalid+invalid", - "-invalid.01", - "alpha", - "alpha.beta", - "alpha.beta.1", - "alpha.1", - "alpha+beta", - "alpha_beta", - "alpha.", - "alpha..", - "beta", - "1.0.0-alpha_beta", - "-alpha.", - "1.0.0-alpha..", - "1.0.0-alpha..1", - "1.0.0-alpha...1", - "1.0.0-alpha....1", - "1.0.0-alpha.....1", - "1.0.0-alpha......1", - "1.0.0-alpha.......1", - "01.1.1", - "1.01.1", - "1.1.01", - "1.2", - "1.2.3.DEV", - "1.2-SNAPSHOT", - "1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788", - "1.2-RC-SNAPSHOT", - "-1.0.3-gamma+b7718", - "+justmeta", - "9.8.7+meta+meta", - "9.8.7-whatever+meta+meta", - "99999999999999999999999.999999999999999999.99999999999999999----RC-SNAPSHOT.12.09.1--------------------------------..12", - ] - ) - ) - - def test__reduce_versions(self): - self.assertVersionSetResult(["0.0.1", "0.0.1"], ["=0.0.1", "=0.0.1"]) - - self.assertVersionSetResult(["0.0.1"], ["=0.0.1", "=0.0.1"]) - - self.assertVersionSetResult([">0.0.1"], [">0.0.1", None]) - - self.assertVersionSetResult(["<0.0.1"], [None, "<0.0.1"]) - - self.assertVersionSetResult([">0.0.1", "0.0.2"], ["=0.0.2", "=0.0.2"]) - - self.assertVersionSetResult(["0.0.2", ">=0.0.2"], ["=0.0.2", "=0.0.2"]) - - self.assertVersionSetResult([">0.0.1", ">0.0.2", ">0.0.3"], [">0.0.3", None]) - - self.assertVersionSetResult([">0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"]) - - self.assertVersionSetResult([">0.0.1", "0.0.2", "<0.0.3"], ["=0.0.2", "=0.0.2"]) - - self.assertVersionSetResult([">0.0.1", ">=0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"]) - - self.assertVersionSetResult([">0.0.1", "<0.0.3", "<=0.0.3"], [">0.0.1", "<0.0.3"]) - - self.assertVersionSetResult([">0.0.1", ">0.0.2", "<0.0.3", "<0.0.4"], [">0.0.2", "<0.0.3"]) - - self.assertVersionSetResult(["<=0.0.3", ">=0.0.3"], [">=0.0.3", "<=0.0.3"]) - - self.assertInvalidVersionSet([">0.0.2", "0.0.1"]) - self.assertInvalidVersionSet([">0.0.2", "0.0.2"]) - self.assertInvalidVersionSet(["<0.0.2", "0.0.2"]) - self.assertInvalidVersionSet(["<0.0.2", ">0.0.3"]) - self.assertInvalidVersionSet(["<=0.0.3", ">0.0.3"]) - self.assertInvalidVersionSet(["<0.0.3", ">=0.0.3"]) - self.assertInvalidVersionSet(["<0.0.3", ">0.0.3"]) - - def test__resolve_to_specific_version(self): - self.assertEqual( - resolve_to_specific_version(create_range(">0.0.1", None), ["0.0.1", "0.0.2"]), "0.0.2" - ) - - self.assertEqual( - resolve_to_specific_version(create_range(">=0.0.2", None), ["0.0.1", "0.0.2"]), "0.0.2" - ) - - self.assertEqual( - resolve_to_specific_version(create_range(">=0.0.3", None), ["0.0.1", "0.0.2"]), None - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=0.0.3", "<0.0.5"), ["0.0.3", "0.0.4", "0.0.5"] - ), - "0.0.4", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(None, "<=0.0.5"), ["0.0.3", "0.1.4", "0.0.5"] - ), - "0.0.5", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range("=0.4.5a2", "=0.4.5a2"), ["0.4.5a1", "0.4.5a2"] - ), - "0.4.5a2", - ) - - self.assertEqual( - resolve_to_specific_version(create_range("=0.7.6", "=0.7.6"), ["0.7.6-b1", "0.7.6"]), - "0.7.6", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"] - ), - "1.2.0a1", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", "<1.2.0"), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"] - ), - "1.1.0", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"] - ), - "1.2.0", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", "<1.2.0"), - ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"], - ), - "1.1.0", - ) - - self.assertEqual( - resolve_to_specific_version( - # https://github.com/dbt-labs/dbt-core/issues/7039 - # 10 is greater than 9 - create_range(">0.9.0", "<0.10.0"), - ["0.9.0", "0.9.1", "0.10.0"], - ), - "0.9.1", - ) - - def test__filter_installable(self): - installable = filter_installable( - [ - "1.1.0", - "1.2.0a1", - "1.0.0", - "2.1.0-alpha", - "2.2.0asdf", - "2.1.0", - "2.2.0", - "2.2.0-fishtown-beta", - "2.2.0-2", - ], - install_prerelease=True, - ) - expected = [ - "1.0.0", - "1.1.0", - "1.2.0a1", - "2.1.0-alpha", - "2.1.0", - "2.2.0-2", - "2.2.0asdf", - "2.2.0-fishtown-beta", - "2.2.0", - ] - assert installable == expected - - installable = filter_installable( - [ - "1.1.0", - "1.2.0a1", - "1.0.0", - "2.1.0-alpha", - "2.2.0asdf", - "2.1.0", - "2.2.0", - "2.2.0-fishtown-beta", - ], - install_prerelease=False, - ) - expected = ["1.0.0", "1.1.0", "2.1.0", "2.2.0"] - assert installable == expected