From 1c3d460d3b41c4898ffeaf9096b0b2443950dbf5 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Mon, 20 May 2024 15:28:02 -0700 Subject: [PATCH 01/10] move test_jinja and convert --- tests/unit/{ => clients}/test_jinja.py | 68 +++++++++++++------------- 1 file changed, 34 insertions(+), 34 deletions(-) rename tests/unit/{ => clients}/test_jinja.py (85%) diff --git a/tests/unit/test_jinja.py b/tests/unit/clients/test_jinja.py similarity index 85% rename from tests/unit/test_jinja.py rename to tests/unit/clients/test_jinja.py index 94150be8e26..5f15d9e3f44 100644 --- a/tests/unit/test_jinja.py +++ b/tests/unit/clients/test_jinja.py @@ -1,4 +1,3 @@ -import unittest from contextlib import contextmanager import pytest @@ -367,7 +366,7 @@ def expected_id(arg): @pytest.mark.parametrize("value,text_expectation,native_expectation", jinja_tests, ids=expected_id) -def test_jinja_rendering(value, text_expectation, native_expectation): +def test_jinja_rendering_string(value, text_expectation, native_expectation): foo_value = yaml.safe_load(value)["foo"] ctx = {"a_str": "100", "a_int": 100, "b_str": "hello"} with text_expectation as text_result: @@ -377,40 +376,41 @@ def test_jinja_rendering(value, text_expectation, native_expectation): assert native_result == get_rendered(foo_value, ctx, native=True) -class TestJinja(unittest.TestCase): - def test_do(self): - s = "{% set my_dict = {} %}\n{% do my_dict.update(a=1) %}" +def test_do(): + s = "{% set my_dict = {} %}\n{% do my_dict.update(a=1) %}" - template = get_template(s, {}) - mod = template.make_module() - self.assertEqual(mod.my_dict, {"a": 1}) + template = get_template(s, {}) + mod = template.make_module() + assert mod.my_dict == {"a": 1} - def test_regular_render(self): - s = '{{ "some_value" | as_native }}' - value = get_rendered(s, {}, native=False) - assert value == "some_value" - s = "{{ 1991 | as_native }}" - value = get_rendered(s, {}, native=False) - assert value == "1991" - s = '{{ "some_value" | as_text }}' - value = get_rendered(s, {}, native=False) - assert value == "some_value" - s = "{{ 1991 | as_text }}" - value = get_rendered(s, {}, native=False) - assert value == "1991" +def test_regular_render(): + s = '{{ "some_value" | as_native }}' + value = get_rendered(s, {}, native=False) + assert value == "some_value" + s = "{{ 1991 | as_native }}" + value = get_rendered(s, {}, native=False) + assert value == "1991" - def test_native_render(self): - s = '{{ "some_value" | as_native }}' - value = get_rendered(s, {}, native=True) - assert value == "some_value" - s = "{{ 1991 | as_native }}" - value = get_rendered(s, {}, native=True) - assert value == 1991 + s = '{{ "some_value" | as_text }}' + value = get_rendered(s, {}, native=False) + assert value == "some_value" + s = "{{ 1991 | as_text }}" + value = get_rendered(s, {}, native=False) + assert value == "1991" - s = '{{ "some_value" | as_text }}' - value = get_rendered(s, {}, native=True) - assert value == "some_value" - s = "{{ 1991 | as_text }}" - value = get_rendered(s, {}, native=True) - assert value == "1991" + +def test_native_render(): + s = '{{ "some_value" | as_native }}' + value = get_rendered(s, {}, native=True) + assert value == "some_value" + s = "{{ 1991 | as_native }}" + value = get_rendered(s, {}, native=True) + assert value == 1991 + + s = '{{ "some_value" | as_text }}' + value = get_rendered(s, {}, native=True) + assert value == "some_value" + s = "{{ 1991 | as_text }}" + value = get_rendered(s, {}, native=True) + assert value == "1991" From fff9a8481ae1f200aa4f85f8b647ef8343bf5dcf Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Mon, 20 May 2024 15:47:28 -0700 Subject: [PATCH 02/10] two more --- tests/unit/{test_base_context.py => context/test_base.py} | 0 .../graph/test_nodes.py} | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) rename tests/unit/{test_base_context.py => context/test_base.py} (100%) rename tests/unit/{test_contracts_graph_compiled.py => contracts/graph/test_nodes.py} (99%) diff --git a/tests/unit/test_base_context.py b/tests/unit/context/test_base.py similarity index 100% rename from tests/unit/test_base_context.py rename to tests/unit/context/test_base.py diff --git a/tests/unit/test_contracts_graph_compiled.py b/tests/unit/contracts/graph/test_nodes.py similarity index 99% rename from tests/unit/test_contracts_graph_compiled.py rename to tests/unit/contracts/graph/test_nodes.py index e0228cc28f6..67eed2903d2 100644 --- a/tests/unit/test_contracts_graph_compiled.py +++ b/tests/unit/contracts/graph/test_nodes.py @@ -8,8 +8,7 @@ from dbt.contracts.graph.nodes import DependsOn, GenericTestNode, ModelConfig, ModelNode from dbt.node_types import NodeType from tests.unit.fixtures import generic_test_node, model_node - -from .utils import ( +from tests.unit.utils import ( assert_fails_validation, assert_from_dict, assert_symmetric, From d28d816f7ad82e246a89aabf46807e1623893abc Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Mon, 20 May 2024 16:09:55 -0700 Subject: [PATCH 03/10] more --- tests/unit/context/__init__.py | 0 .../graph/test_node_args.py} | 0 .../graph/test_nodes_parsed.py} | 3 +-- tests/unit/task/__init__.py | 0 4 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 tests/unit/context/__init__.py rename tests/unit/{test_contracts_graph_node_args.py => contracts/graph/test_node_args.py} (100%) rename tests/unit/{test_contracts_graph_parsed.py => contracts/graph/test_nodes_parsed.py} (99%) create mode 100644 tests/unit/task/__init__.py diff --git a/tests/unit/context/__init__.py b/tests/unit/context/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_contracts_graph_node_args.py b/tests/unit/contracts/graph/test_node_args.py similarity index 100% rename from tests/unit/test_contracts_graph_node_args.py rename to tests/unit/contracts/graph/test_node_args.py diff --git a/tests/unit/test_contracts_graph_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py similarity index 99% rename from tests/unit/test_contracts_graph_parsed.py rename to tests/unit/contracts/graph/test_nodes_parsed.py index 7a62c394b22..c0ed94b430c 100644 --- a/tests/unit/test_contracts_graph_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -56,8 +56,7 @@ ) from dbt.node_types import AccessType, NodeType from dbt_common.dataclass_schema import ValidationError - -from .utils import ( +from tests.unit.utils import ( ContractTestCase, assert_fails_validation, assert_from_dict, diff --git a/tests/unit/task/__init__.py b/tests/unit/task/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 3beb5b5940b23ff30797daf16b70c790dac26490 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Mon, 20 May 2024 16:12:27 -0700 Subject: [PATCH 04/10] more --- tests/unit/contracts/graph/test_unparsed.py | 989 +++++++++++++++++- tests/unit/test_contracts_graph_unparsed.py | 986 ----------------- ...tions.py => test_internal_deprecations.py} | 0 3 files changed, 980 insertions(+), 995 deletions(-) delete mode 100644 tests/unit/test_contracts_graph_unparsed.py rename tests/unit/{test_deprecations.py => test_internal_deprecations.py} (100%) diff --git a/tests/unit/contracts/graph/test_unparsed.py b/tests/unit/contracts/graph/test_unparsed.py index 65d29ad0947..0368e9c2770 100644 --- a/tests/unit/contracts/graph/test_unparsed.py +++ b/tests/unit/contracts/graph/test_unparsed.py @@ -1,14 +1,985 @@ +import pickle +from datetime import timedelta + import pytest -from dbt.contracts.graph.unparsed import HasColumnTests, UnparsedColumn -from dbt.exceptions import ParsingError -from dbt.parser.schemas import ParserRef +from dbt.artifacts.resources import ( + ExposureType, + FreshnessThreshold, + MaturityType, + Owner, + Quoting, + Time, +) +from dbt.artifacts.resources.types import TimePeriod +from dbt.artifacts.schemas.results import FreshnessStatus +from dbt.contracts.graph.unparsed import ( + Docs, + UnparsedColumn, + UnparsedDocumentationFile, + UnparsedExposure, + UnparsedMacro, + UnparsedMetric, + UnparsedMetricInputMeasure, + UnparsedMetricTypeParams, + UnparsedModelUpdate, + UnparsedNode, + UnparsedNodeUpdate, + UnparsedRunHook, + UnparsedSourceDefinition, + UnparsedSourceTableDefinition, + UnparsedVersion, +) +from dbt.node_types import NodeType +from tests.unit.utils import ContractTestCase + + +class TestUnparsedMacro(ContractTestCase): + ContractType = UnparsedMacro + + def test_ok(self): + macro_dict = { + "path": "/root/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", + "resource_type": "macro", + } + macro = self.ContractType( + path="/root/path.sql", + original_file_path="/root/path.sql", + package_name="test", + language="sql", + raw_code="{% macro foo() %}select 1 as id{% endmacro %}", + resource_type=NodeType.Macro, + ) + self.assert_symmetric(macro, macro_dict) + pickle.loads(pickle.dumps(macro)) + + def test_invalid_missing_field(self): + macro_dict = { + "path": "/root/path.sql", + "original_file_path": "/root/path.sql", + # 'package_name': 'test', + "language": "sql", + "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", + "resource_type": "macro", + } + self.assert_fails_validation(macro_dict) + + def test_invalid_extra_field(self): + macro_dict = { + "path": "/root/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", + "extra": "extra", + "resource_type": "macro", + } + self.assert_fails_validation(macro_dict) + + +class TestUnparsedNode(ContractTestCase): + ContractType = UnparsedNode + + def test_ok(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Model, + "path": "/root/x/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": 'select * from {{ ref("thing") }}', + } + node = self.ContractType( + package_name="test", + path="/root/x/path.sql", + original_file_path="/root/path.sql", + language="sql", + raw_code='select * from {{ ref("thing") }}', + name="foo", + resource_type=NodeType.Model, + ) + self.assert_symmetric(node, node_dict) + self.assertFalse(node.empty) + + self.assert_fails_validation(node_dict, cls=UnparsedRunHook) + self.assert_fails_validation(node_dict, cls=UnparsedMacro) + pickle.loads(pickle.dumps(node)) + + def test_empty(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Model, + "path": "/root/x/path.sql", + "original_file_path": "/root/path.sql", + "package_name": "test", + "language": "sql", + "raw_code": " \n", + } + node = UnparsedNode( + package_name="test", + path="/root/x/path.sql", + original_file_path="/root/path.sql", + language="sql", + raw_code=" \n", + name="foo", + resource_type=NodeType.Model, + ) + self.assert_symmetric(node, node_dict) + self.assertTrue(node.empty) + + self.assert_fails_validation(node_dict, cls=UnparsedRunHook) + self.assert_fails_validation(node_dict, cls=UnparsedMacro) + + +class TestUnparsedRunHook(ContractTestCase): + ContractType = UnparsedRunHook + + def test_ok(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Operation, + "path": "/root/dbt_project.yml", + "original_file_path": "/root/dbt_project.yml", + "package_name": "test", + "language": "sql", + "raw_code": "GRANT select on dbt_postgres", + "index": 4, + } + node = self.ContractType( + package_name="test", + path="/root/dbt_project.yml", + original_file_path="/root/dbt_project.yml", + language="sql", + raw_code="GRANT select on dbt_postgres", + name="foo", + resource_type=NodeType.Operation, + index=4, + ) + self.assert_symmetric(node, node_dict) + self.assert_fails_validation(node_dict, cls=UnparsedNode) + pickle.loads(pickle.dumps(node)) + + def test_bad_type(self): + node_dict = { + "name": "foo", + "resource_type": NodeType.Model, # invalid + "path": "/root/dbt_project.yml", + "original_file_path": "/root/dbt_project.yml", + "package_name": "test", + "language": "sql", + "raw_code": "GRANT select on dbt_postgres", + "index": 4, + } + self.assert_fails_validation(node_dict) + + +class TestFreshnessThreshold(ContractTestCase): + ContractType = FreshnessThreshold + + def test_empty(self): + empty = self.ContractType() + self.assert_symmetric(empty, {"error_after": {}, "warn_after": {}}) + self.assertEqual(empty.status(float("Inf")), FreshnessStatus.Pass) + self.assertEqual(empty.status(0), FreshnessStatus.Pass) + + def test_both(self): + threshold = self.ContractType( + warn_after=Time(count=18, period=TimePeriod.hour), + error_after=Time(count=2, period=TimePeriod.day), + ) + dct = { + "error_after": {"count": 2, "period": "day"}, + "warn_after": {"count": 18, "period": "hour"}, + } + self.assert_symmetric(threshold, dct) + + error_seconds = timedelta(days=3).total_seconds() + warn_seconds = timedelta(days=1).total_seconds() + pass_seconds = timedelta(hours=3).total_seconds() + self.assertEqual(threshold.status(error_seconds), FreshnessStatus.Error) + self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) + self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) + pickle.loads(pickle.dumps(threshold)) + + def test_merged(self): + t1 = self.ContractType( + warn_after=Time(count=36, period=TimePeriod.hour), + error_after=Time(count=2, period=TimePeriod.day), + ) + t2 = self.ContractType( + warn_after=Time(count=18, period=TimePeriod.hour), + ) + threshold = self.ContractType( + warn_after=Time(count=18, period=TimePeriod.hour), + error_after=Time(count=None, period=None), + ) + self.assertEqual(threshold, t1.merged(t2)) + + warn_seconds = timedelta(days=1).total_seconds() + pass_seconds = timedelta(hours=3).total_seconds() + self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) + self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) + + +class TestQuoting(ContractTestCase): + ContractType = Quoting + + def test_empty(self): + empty = self.ContractType() + self.assert_symmetric(empty, {}) + + def test_partial(self): + a = self.ContractType(None, True, False) + b = self.ContractType(True, False, None) + self.assert_symmetric(a, {"schema": True, "identifier": False}) + self.assert_symmetric(b, {"database": True, "schema": False}) + + c = a.merged(b) + self.assertEqual(c, self.ContractType(True, False, False)) + self.assert_symmetric(c, {"database": True, "schema": False, "identifier": False}) + pickle.loads(pickle.dumps(c)) + + +class TestUnparsedSourceDefinition(ContractTestCase): + ContractType = UnparsedSourceDefinition + + def test_defaults(self): + minimum = self.ContractType(name="foo") + from_dict = {"name": "foo"} + to_dict = { + "name": "foo", + "description": "", + "freshness": {"error_after": {}, "warn_after": {}}, + "quoting": {}, + "tables": [], + "loader": "", + "meta": {}, + "tags": [], + "config": {}, + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) + + def test_contents(self): + empty = self.ContractType( + name="foo", + description="a description", + quoting=Quoting(database=False), + loader="some_loader", + freshness=FreshnessThreshold(), + tables=[], + meta={}, + ) + dct = { + "name": "foo", + "description": "a description", + "quoting": {"database": False}, + "loader": "some_loader", + "freshness": {"error_after": {}, "warn_after": {}}, + "tables": [], + "meta": {}, + "tags": [], + "config": {}, + } + self.assert_symmetric(empty, dct) + + def test_table_defaults(self): + table_1 = UnparsedSourceTableDefinition(name="table1") + table_2 = UnparsedSourceTableDefinition( + name="table2", + description="table 2", + quoting=Quoting(database=True), + ) + source = self.ContractType(name="foo", tables=[table_1, table_2]) + from_dict = { + "name": "foo", + "tables": [ + {"name": "table1"}, + { + "name": "table2", + "description": "table 2", + "quoting": {"database": True}, + }, + ], + } + to_dict = { + "name": "foo", + "description": "", + "config": {}, + "loader": "", + "freshness": {"error_after": {}, "warn_after": {}}, + "quoting": {}, + "meta": {}, + "tables": [ + { + "name": "table1", + "description": "", + "config": {}, + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "columns": [], + "constraints": [], + "quoting": {}, + "freshness": {"error_after": {}, "warn_after": {}}, + "meta": {}, + "tags": [], + }, + { + "name": "table2", + "description": "table 2", + "config": {}, + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "columns": [], + "constraints": [], + "quoting": {"database": True}, + "freshness": {"error_after": {}, "warn_after": {}}, + "meta": {}, + "tags": [], + }, + ], + "tags": [], + } + self.assert_from_dict(source, from_dict) + self.assert_symmetric(source, to_dict) + pickle.loads(pickle.dumps(source)) + + +class TestUnparsedDocumentationFile(ContractTestCase): + ContractType = UnparsedDocumentationFile + + def test_ok(self): + doc = self.ContractType( + package_name="test", + path="/root/docs", + original_file_path="/root/docs/doc.md", + file_contents="blah blah blah", + ) + doc_dict = { + "package_name": "test", + "path": "/root/docs", + "original_file_path": "/root/docs/doc.md", + "file_contents": "blah blah blah", + } + self.assert_symmetric(doc, doc_dict) + self.assertEqual(doc.resource_type, NodeType.Documentation) + self.assert_fails_validation(doc_dict, UnparsedNode) + pickle.loads(pickle.dumps(doc)) + + def test_extra_field(self): + self.assert_fails_validation({}) + doc_dict = { + "package_name": "test", + "path": "/root/docs", + "original_file_path": "/root/docs/doc.md", + "file_contents": "blah blah blah", + "resource_type": "docs", + } + self.assert_fails_validation(doc_dict) + + +class TestUnparsedNodeUpdate(ContractTestCase): + ContractType = UnparsedNodeUpdate + + def test_defaults(self): + minimum = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + ) + from_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + } + to_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "columns": [], + "description": "", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {}, + "config": {}, + "constraints": [], + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) + + def test_contents(self): + update = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + description="a description", + data_tests=["table_test"], + meta={"key": ["value1", "value2"]}, + columns=[ + UnparsedColumn( + name="x", + description="x description", + meta={"key2": "value3"}, + ), + UnparsedColumn( + name="y", + description="y description", + data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], + meta={}, + tags=["a", "b"], + ), + ], + docs=Docs(show=False), + ) + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "constraints": [], + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + "tags": [], + "constraints": [], + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "tags": ["a", "b"], + "constraints": [], + }, + ], + "docs": {"show": False}, + "config": {}, + } + self.assert_symmetric(update, dct) + pickle.loads(pickle.dumps(update)) + + def test_bad_test_type(self): + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + # column missing a name + { + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + # missing a name + dct = { + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + +class TestUnparsedModelUpdate(ContractTestCase): + ContractType = UnparsedModelUpdate + + def test_defaults(self): + minimum = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + ) + from_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + } + to_dict = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "columns": [], + "description": "", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {}, + "config": {}, + "constraints": [], + "versions": [], + } + self.assert_from_dict(minimum, from_dict) + self.assert_to_dict(minimum, to_dict) + + def test_contents(self): + update = self.ContractType( + name="foo", + yaml_key="models", + original_file_path="/some/fake/path", + package_name="test", + description="a description", + data_tests=["table_test"], + meta={"key": ["value1", "value2"]}, + columns=[ + UnparsedColumn( + name="x", + description="x description", + meta={"key2": "value3"}, + ), + UnparsedColumn( + name="y", + description="y description", + data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], + meta={}, + tags=["a", "b"], + ), + ], + docs=Docs(show=False), + versions=[UnparsedVersion(v=2)], + ) + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "constraints": [], + "versions": [ + { + "v": 2, + "description": "", + "columns": [], + "config": {}, + "constraints": [], + "docs": {"show": True}, + } + ], + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + "tags": [], + "constraints": [], + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "tags": ["a", "b"], + "constraints": [], + }, + ], + "docs": {"show": False}, + "config": {}, + } + self.assert_symmetric(update, dct) + pickle.loads(pickle.dumps(update)) + + def test_bad_test_type(self): + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + dct = { + "name": "foo", + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + # column missing a name + { + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + # missing a name + dct = { + "yaml_key": "models", + "original_file_path": "/some/fake/path", + "package_name": "test", + "description": "a description", + "data_tests": ["table_test"], + "tests": [], + "meta": {"key": ["value1", "value2"]}, + "columns": [ + { + "name": "x", + "description": "x description", + "docs": {"show": True}, + "data_tests": [], + "tests": [], + "meta": {"key2": "value3"}, + }, + { + "name": "y", + "description": "y description", + "docs": {"show": True}, + "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], + "tests": [], + "meta": {}, + "yaml_key": "models", + "original_file_path": "/some/fake/path", + }, + ], + "docs": {"show": True}, + } + self.assert_fails_validation(dct) + + +class TestUnparsedExposure(ContractTestCase): + ContractType = UnparsedExposure + + def get_ok_dict(self): + return { + "name": "my_exposure", + "type": "dashboard", + "owner": {"name": "example", "email": "name@example.com", "slack": "#channel"}, + "maturity": "medium", + "meta": {"tool": "my_tool"}, + "tags": ["my_department"], + "url": "https://example.com/dashboards/1", + "description": "A exposure", + "config": {}, + "depends_on": [ + 'ref("my_model")', + 'source("raw", "source_table")', + ], + } + + def test_ok(self): + exposure = self.ContractType( + name="my_exposure", + type=ExposureType.Dashboard, + owner=Owner(name="example", email="name@example.com", _extra={"slack": "#channel"}), + maturity=MaturityType.Medium, + url="https://example.com/dashboards/1", + description="A exposure", + config={}, + meta={"tool": "my_tool"}, + tags=["my_department"], + depends_on=['ref("my_model")', 'source("raw", "source_table")'], + ) + dct = self.get_ok_dict() + self.assert_symmetric(exposure, dct) + pickle.loads(pickle.dumps(exposure)) + + def test_ok_exposures(self): + for exposure_allowed in ("dashboard", "notebook", "analysis", "ml", "application"): + tst = self.get_ok_dict() + tst["type"] = exposure_allowed + assert self.ContractType.from_dict(tst).type == exposure_allowed + + def test_bad_exposure(self): + # bad exposure: None isn't allowed + for exposure_not_allowed in (None, "not an exposure"): + tst = self.get_ok_dict() + tst["type"] = exposure_not_allowed + self.assert_fails_validation(tst) + + def test_no_exposure(self): + tst = self.get_ok_dict() + del tst["type"] + self.assert_fails_validation(tst) + + def test_ok_maturities(self): + for maturity_allowed in (None, "low", "medium", "high"): + tst = self.get_ok_dict() + tst["maturity"] = maturity_allowed + assert self.ContractType.from_dict(tst).maturity == maturity_allowed + + tst = self.get_ok_dict() + del tst["maturity"] + assert self.ContractType.from_dict(tst).maturity is None + + def test_bad_maturity(self): + tst = self.get_ok_dict() + tst["maturity"] = "invalid maturity" + self.assert_fails_validation(tst) + + def test_bad_owner_missing_things(self): + tst = self.get_ok_dict() + del tst["owner"]["email"] + del tst["owner"]["name"] + self.assert_fails_validation(tst) + + del tst["owner"] + self.assert_fails_validation(tst) + + def test_bad_tags(self): + tst = self.get_ok_dict() + tst["tags"] = [123] + self.assert_fails_validation(tst) + + +class TestUnparsedMetric(ContractTestCase): + ContractType = UnparsedMetric + + def get_ok_dict(self): + return { + "name": "new_customers", + "label": "New Customers", + "description": "New customers", + "type": "simple", + "type_params": { + "measure": { + "name": "customers", + "filter": "is_new = true", + "join_to_timespine": False, + }, + }, + "config": {}, + "tags": [], + "meta": {"is_okr": True}, + } + + def test_ok(self): + metric = self.ContractType( + name="new_customers", + label="New Customers", + description="New customers", + type="simple", + type_params=UnparsedMetricTypeParams( + measure=UnparsedMetricInputMeasure( + name="customers", + filter="is_new = true", + ) + ), + config={}, + meta={"is_okr": True}, + ) + dct = self.get_ok_dict() + self.assert_symmetric(metric, dct) + pickle.loads(pickle.dumps(metric)) + + def test_bad_metric_no_type_params(self): + tst = self.get_ok_dict() + del tst["type_params"] + self.assert_fails_validation(tst) + + def test_bad_tags(self): + tst = self.get_ok_dict() + tst["tags"] = [123] + self.assert_fails_validation(tst) + + +class TestUnparsedVersion(ContractTestCase): + ContractType = UnparsedVersion + + def get_ok_dict(self): + return { + "v": 2, + "defined_in": "test_defined_in", + "description": "A version", + "config": {}, + "constraints": [], + "docs": {"show": False}, + "data_tests": [], + "columns": [], + } + + def test_ok(self): + version = self.ContractType( + v=2, + defined_in="test_defined_in", + description="A version", + config={}, + constraints=[], + docs=Docs(show=False), + data_tests=[], + columns=[], + ) + dct = self.get_ok_dict() + self.assert_symmetric(version, dct) + pickle.loads(pickle.dumps(version)) + def test_bad_version_no_v(self): + version = self.get_ok_dict() + del version["v"] + self.assert_fails_validation(version) -def test_column_parse(): - unparsed_col = HasColumnTests( - columns=[UnparsedColumn(name="TestCol", constraints=[{"type": "!INVALID!"}])] - ) - with pytest.raises(ParsingError): - ParserRef.from_target(unparsed_col) +@pytest.mark.parametrize( + "left,right,expected_lt", + [ + # same types + (2, 12, True), + (12, 2, False), + ("a", "b", True), + ("b", "a", False), + # mismatched types - numeric + (2, 12.0, True), + (12.0, 2, False), + (2, "12", True), + ("12", 2, False), + # mismatched types + (1, "test", True), + ("test", 1, False), + ], +) +def test_unparsed_version_lt(left, right, expected_lt): + assert (UnparsedVersion(left) < UnparsedVersion(right)) == expected_lt diff --git a/tests/unit/test_contracts_graph_unparsed.py b/tests/unit/test_contracts_graph_unparsed.py deleted file mode 100644 index e1105b80820..00000000000 --- a/tests/unit/test_contracts_graph_unparsed.py +++ /dev/null @@ -1,986 +0,0 @@ -import pickle -from datetime import timedelta - -import pytest - -from dbt.artifacts.resources import ( - ExposureType, - FreshnessThreshold, - MaturityType, - Owner, - Quoting, - Time, -) -from dbt.artifacts.resources.types import TimePeriod -from dbt.artifacts.schemas.results import FreshnessStatus -from dbt.contracts.graph.unparsed import ( - Docs, - UnparsedColumn, - UnparsedDocumentationFile, - UnparsedExposure, - UnparsedMacro, - UnparsedMetric, - UnparsedMetricInputMeasure, - UnparsedMetricTypeParams, - UnparsedModelUpdate, - UnparsedNode, - UnparsedNodeUpdate, - UnparsedRunHook, - UnparsedSourceDefinition, - UnparsedSourceTableDefinition, - UnparsedVersion, -) -from dbt.node_types import NodeType - -from .utils import ContractTestCase - - -class TestUnparsedMacro(ContractTestCase): - ContractType = UnparsedMacro - - def test_ok(self): - macro_dict = { - "path": "/root/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", - "resource_type": "macro", - } - macro = self.ContractType( - path="/root/path.sql", - original_file_path="/root/path.sql", - package_name="test", - language="sql", - raw_code="{% macro foo() %}select 1 as id{% endmacro %}", - resource_type=NodeType.Macro, - ) - self.assert_symmetric(macro, macro_dict) - pickle.loads(pickle.dumps(macro)) - - def test_invalid_missing_field(self): - macro_dict = { - "path": "/root/path.sql", - "original_file_path": "/root/path.sql", - # 'package_name': 'test', - "language": "sql", - "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", - "resource_type": "macro", - } - self.assert_fails_validation(macro_dict) - - def test_invalid_extra_field(self): - macro_dict = { - "path": "/root/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": "{% macro foo() %}select 1 as id{% endmacro %}", - "extra": "extra", - "resource_type": "macro", - } - self.assert_fails_validation(macro_dict) - - -class TestUnparsedNode(ContractTestCase): - ContractType = UnparsedNode - - def test_ok(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Model, - "path": "/root/x/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": 'select * from {{ ref("thing") }}', - } - node = self.ContractType( - package_name="test", - path="/root/x/path.sql", - original_file_path="/root/path.sql", - language="sql", - raw_code='select * from {{ ref("thing") }}', - name="foo", - resource_type=NodeType.Model, - ) - self.assert_symmetric(node, node_dict) - self.assertFalse(node.empty) - - self.assert_fails_validation(node_dict, cls=UnparsedRunHook) - self.assert_fails_validation(node_dict, cls=UnparsedMacro) - pickle.loads(pickle.dumps(node)) - - def test_empty(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Model, - "path": "/root/x/path.sql", - "original_file_path": "/root/path.sql", - "package_name": "test", - "language": "sql", - "raw_code": " \n", - } - node = UnparsedNode( - package_name="test", - path="/root/x/path.sql", - original_file_path="/root/path.sql", - language="sql", - raw_code=" \n", - name="foo", - resource_type=NodeType.Model, - ) - self.assert_symmetric(node, node_dict) - self.assertTrue(node.empty) - - self.assert_fails_validation(node_dict, cls=UnparsedRunHook) - self.assert_fails_validation(node_dict, cls=UnparsedMacro) - - -class TestUnparsedRunHook(ContractTestCase): - ContractType = UnparsedRunHook - - def test_ok(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Operation, - "path": "/root/dbt_project.yml", - "original_file_path": "/root/dbt_project.yml", - "package_name": "test", - "language": "sql", - "raw_code": "GRANT select on dbt_postgres", - "index": 4, - } - node = self.ContractType( - package_name="test", - path="/root/dbt_project.yml", - original_file_path="/root/dbt_project.yml", - language="sql", - raw_code="GRANT select on dbt_postgres", - name="foo", - resource_type=NodeType.Operation, - index=4, - ) - self.assert_symmetric(node, node_dict) - self.assert_fails_validation(node_dict, cls=UnparsedNode) - pickle.loads(pickle.dumps(node)) - - def test_bad_type(self): - node_dict = { - "name": "foo", - "resource_type": NodeType.Model, # invalid - "path": "/root/dbt_project.yml", - "original_file_path": "/root/dbt_project.yml", - "package_name": "test", - "language": "sql", - "raw_code": "GRANT select on dbt_postgres", - "index": 4, - } - self.assert_fails_validation(node_dict) - - -class TestFreshnessThreshold(ContractTestCase): - ContractType = FreshnessThreshold - - def test_empty(self): - empty = self.ContractType() - self.assert_symmetric(empty, {"error_after": {}, "warn_after": {}}) - self.assertEqual(empty.status(float("Inf")), FreshnessStatus.Pass) - self.assertEqual(empty.status(0), FreshnessStatus.Pass) - - def test_both(self): - threshold = self.ContractType( - warn_after=Time(count=18, period=TimePeriod.hour), - error_after=Time(count=2, period=TimePeriod.day), - ) - dct = { - "error_after": {"count": 2, "period": "day"}, - "warn_after": {"count": 18, "period": "hour"}, - } - self.assert_symmetric(threshold, dct) - - error_seconds = timedelta(days=3).total_seconds() - warn_seconds = timedelta(days=1).total_seconds() - pass_seconds = timedelta(hours=3).total_seconds() - self.assertEqual(threshold.status(error_seconds), FreshnessStatus.Error) - self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) - self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) - pickle.loads(pickle.dumps(threshold)) - - def test_merged(self): - t1 = self.ContractType( - warn_after=Time(count=36, period=TimePeriod.hour), - error_after=Time(count=2, period=TimePeriod.day), - ) - t2 = self.ContractType( - warn_after=Time(count=18, period=TimePeriod.hour), - ) - threshold = self.ContractType( - warn_after=Time(count=18, period=TimePeriod.hour), - error_after=Time(count=None, period=None), - ) - self.assertEqual(threshold, t1.merged(t2)) - - warn_seconds = timedelta(days=1).total_seconds() - pass_seconds = timedelta(hours=3).total_seconds() - self.assertEqual(threshold.status(warn_seconds), FreshnessStatus.Warn) - self.assertEqual(threshold.status(pass_seconds), FreshnessStatus.Pass) - - -class TestQuoting(ContractTestCase): - ContractType = Quoting - - def test_empty(self): - empty = self.ContractType() - self.assert_symmetric(empty, {}) - - def test_partial(self): - a = self.ContractType(None, True, False) - b = self.ContractType(True, False, None) - self.assert_symmetric(a, {"schema": True, "identifier": False}) - self.assert_symmetric(b, {"database": True, "schema": False}) - - c = a.merged(b) - self.assertEqual(c, self.ContractType(True, False, False)) - self.assert_symmetric(c, {"database": True, "schema": False, "identifier": False}) - pickle.loads(pickle.dumps(c)) - - -class TestUnparsedSourceDefinition(ContractTestCase): - ContractType = UnparsedSourceDefinition - - def test_defaults(self): - minimum = self.ContractType(name="foo") - from_dict = {"name": "foo"} - to_dict = { - "name": "foo", - "description": "", - "freshness": {"error_after": {}, "warn_after": {}}, - "quoting": {}, - "tables": [], - "loader": "", - "meta": {}, - "tags": [], - "config": {}, - } - self.assert_from_dict(minimum, from_dict) - self.assert_to_dict(minimum, to_dict) - - def test_contents(self): - empty = self.ContractType( - name="foo", - description="a description", - quoting=Quoting(database=False), - loader="some_loader", - freshness=FreshnessThreshold(), - tables=[], - meta={}, - ) - dct = { - "name": "foo", - "description": "a description", - "quoting": {"database": False}, - "loader": "some_loader", - "freshness": {"error_after": {}, "warn_after": {}}, - "tables": [], - "meta": {}, - "tags": [], - "config": {}, - } - self.assert_symmetric(empty, dct) - - def test_table_defaults(self): - table_1 = UnparsedSourceTableDefinition(name="table1") - table_2 = UnparsedSourceTableDefinition( - name="table2", - description="table 2", - quoting=Quoting(database=True), - ) - source = self.ContractType(name="foo", tables=[table_1, table_2]) - from_dict = { - "name": "foo", - "tables": [ - {"name": "table1"}, - { - "name": "table2", - "description": "table 2", - "quoting": {"database": True}, - }, - ], - } - to_dict = { - "name": "foo", - "description": "", - "config": {}, - "loader": "", - "freshness": {"error_after": {}, "warn_after": {}}, - "quoting": {}, - "meta": {}, - "tables": [ - { - "name": "table1", - "description": "", - "config": {}, - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "columns": [], - "constraints": [], - "quoting": {}, - "freshness": {"error_after": {}, "warn_after": {}}, - "meta": {}, - "tags": [], - }, - { - "name": "table2", - "description": "table 2", - "config": {}, - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "columns": [], - "constraints": [], - "quoting": {"database": True}, - "freshness": {"error_after": {}, "warn_after": {}}, - "meta": {}, - "tags": [], - }, - ], - "tags": [], - } - self.assert_from_dict(source, from_dict) - self.assert_symmetric(source, to_dict) - pickle.loads(pickle.dumps(source)) - - -class TestUnparsedDocumentationFile(ContractTestCase): - ContractType = UnparsedDocumentationFile - - def test_ok(self): - doc = self.ContractType( - package_name="test", - path="/root/docs", - original_file_path="/root/docs/doc.md", - file_contents="blah blah blah", - ) - doc_dict = { - "package_name": "test", - "path": "/root/docs", - "original_file_path": "/root/docs/doc.md", - "file_contents": "blah blah blah", - } - self.assert_symmetric(doc, doc_dict) - self.assertEqual(doc.resource_type, NodeType.Documentation) - self.assert_fails_validation(doc_dict, UnparsedNode) - pickle.loads(pickle.dumps(doc)) - - def test_extra_field(self): - self.assert_fails_validation({}) - doc_dict = { - "package_name": "test", - "path": "/root/docs", - "original_file_path": "/root/docs/doc.md", - "file_contents": "blah blah blah", - "resource_type": "docs", - } - self.assert_fails_validation(doc_dict) - - -class TestUnparsedNodeUpdate(ContractTestCase): - ContractType = UnparsedNodeUpdate - - def test_defaults(self): - minimum = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - ) - from_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - } - to_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "columns": [], - "description": "", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {}, - "config": {}, - "constraints": [], - } - self.assert_from_dict(minimum, from_dict) - self.assert_to_dict(minimum, to_dict) - - def test_contents(self): - update = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - description="a description", - data_tests=["table_test"], - meta={"key": ["value1", "value2"]}, - columns=[ - UnparsedColumn( - name="x", - description="x description", - meta={"key2": "value3"}, - ), - UnparsedColumn( - name="y", - description="y description", - data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], - meta={}, - tags=["a", "b"], - ), - ], - docs=Docs(show=False), - ) - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "constraints": [], - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - "tags": [], - "constraints": [], - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "tags": ["a", "b"], - "constraints": [], - }, - ], - "docs": {"show": False}, - "config": {}, - } - self.assert_symmetric(update, dct) - pickle.loads(pickle.dumps(update)) - - def test_bad_test_type(self): - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - # column missing a name - { - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - # missing a name - dct = { - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - -class TestUnparsedModelUpdate(ContractTestCase): - ContractType = UnparsedModelUpdate - - def test_defaults(self): - minimum = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - ) - from_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - } - to_dict = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "columns": [], - "description": "", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {}, - "config": {}, - "constraints": [], - "versions": [], - } - self.assert_from_dict(minimum, from_dict) - self.assert_to_dict(minimum, to_dict) - - def test_contents(self): - update = self.ContractType( - name="foo", - yaml_key="models", - original_file_path="/some/fake/path", - package_name="test", - description="a description", - data_tests=["table_test"], - meta={"key": ["value1", "value2"]}, - columns=[ - UnparsedColumn( - name="x", - description="x description", - meta={"key2": "value3"}, - ), - UnparsedColumn( - name="y", - description="y description", - data_tests=["unique", {"accepted_values": {"values": ["blue", "green"]}}], - meta={}, - tags=["a", "b"], - ), - ], - docs=Docs(show=False), - versions=[UnparsedVersion(v=2)], - ) - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "constraints": [], - "versions": [ - { - "v": 2, - "description": "", - "columns": [], - "config": {}, - "constraints": [], - "docs": {"show": True}, - } - ], - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - "tags": [], - "constraints": [], - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "tags": ["a", "b"], - "constraints": [], - }, - ], - "docs": {"show": False}, - "config": {}, - } - self.assert_symmetric(update, dct) - pickle.loads(pickle.dumps(update)) - - def test_bad_test_type(self): - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": [100, {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - dct = { - "name": "foo", - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - # column missing a name - { - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - # missing a name - dct = { - "yaml_key": "models", - "original_file_path": "/some/fake/path", - "package_name": "test", - "description": "a description", - "data_tests": ["table_test"], - "tests": [], - "meta": {"key": ["value1", "value2"]}, - "columns": [ - { - "name": "x", - "description": "x description", - "docs": {"show": True}, - "data_tests": [], - "tests": [], - "meta": {"key2": "value3"}, - }, - { - "name": "y", - "description": "y description", - "docs": {"show": True}, - "data_tests": ["unique", {"accepted_values": {"values": ["blue", "green"]}}], - "tests": [], - "meta": {}, - "yaml_key": "models", - "original_file_path": "/some/fake/path", - }, - ], - "docs": {"show": True}, - } - self.assert_fails_validation(dct) - - -class TestUnparsedExposure(ContractTestCase): - ContractType = UnparsedExposure - - def get_ok_dict(self): - return { - "name": "my_exposure", - "type": "dashboard", - "owner": {"name": "example", "email": "name@example.com", "slack": "#channel"}, - "maturity": "medium", - "meta": {"tool": "my_tool"}, - "tags": ["my_department"], - "url": "https://example.com/dashboards/1", - "description": "A exposure", - "config": {}, - "depends_on": [ - 'ref("my_model")', - 'source("raw", "source_table")', - ], - } - - def test_ok(self): - exposure = self.ContractType( - name="my_exposure", - type=ExposureType.Dashboard, - owner=Owner(name="example", email="name@example.com", _extra={"slack": "#channel"}), - maturity=MaturityType.Medium, - url="https://example.com/dashboards/1", - description="A exposure", - config={}, - meta={"tool": "my_tool"}, - tags=["my_department"], - depends_on=['ref("my_model")', 'source("raw", "source_table")'], - ) - dct = self.get_ok_dict() - self.assert_symmetric(exposure, dct) - pickle.loads(pickle.dumps(exposure)) - - def test_ok_exposures(self): - for exposure_allowed in ("dashboard", "notebook", "analysis", "ml", "application"): - tst = self.get_ok_dict() - tst["type"] = exposure_allowed - assert self.ContractType.from_dict(tst).type == exposure_allowed - - def test_bad_exposure(self): - # bad exposure: None isn't allowed - for exposure_not_allowed in (None, "not an exposure"): - tst = self.get_ok_dict() - tst["type"] = exposure_not_allowed - self.assert_fails_validation(tst) - - def test_no_exposure(self): - tst = self.get_ok_dict() - del tst["type"] - self.assert_fails_validation(tst) - - def test_ok_maturities(self): - for maturity_allowed in (None, "low", "medium", "high"): - tst = self.get_ok_dict() - tst["maturity"] = maturity_allowed - assert self.ContractType.from_dict(tst).maturity == maturity_allowed - - tst = self.get_ok_dict() - del tst["maturity"] - assert self.ContractType.from_dict(tst).maturity is None - - def test_bad_maturity(self): - tst = self.get_ok_dict() - tst["maturity"] = "invalid maturity" - self.assert_fails_validation(tst) - - def test_bad_owner_missing_things(self): - tst = self.get_ok_dict() - del tst["owner"]["email"] - del tst["owner"]["name"] - self.assert_fails_validation(tst) - - del tst["owner"] - self.assert_fails_validation(tst) - - def test_bad_tags(self): - tst = self.get_ok_dict() - tst["tags"] = [123] - self.assert_fails_validation(tst) - - -class TestUnparsedMetric(ContractTestCase): - ContractType = UnparsedMetric - - def get_ok_dict(self): - return { - "name": "new_customers", - "label": "New Customers", - "description": "New customers", - "type": "simple", - "type_params": { - "measure": { - "name": "customers", - "filter": "is_new = true", - "join_to_timespine": False, - }, - }, - "config": {}, - "tags": [], - "meta": {"is_okr": True}, - } - - def test_ok(self): - metric = self.ContractType( - name="new_customers", - label="New Customers", - description="New customers", - type="simple", - type_params=UnparsedMetricTypeParams( - measure=UnparsedMetricInputMeasure( - name="customers", - filter="is_new = true", - ) - ), - config={}, - meta={"is_okr": True}, - ) - dct = self.get_ok_dict() - self.assert_symmetric(metric, dct) - pickle.loads(pickle.dumps(metric)) - - def test_bad_metric_no_type_params(self): - tst = self.get_ok_dict() - del tst["type_params"] - self.assert_fails_validation(tst) - - def test_bad_tags(self): - tst = self.get_ok_dict() - tst["tags"] = [123] - self.assert_fails_validation(tst) - - -class TestUnparsedVersion(ContractTestCase): - ContractType = UnparsedVersion - - def get_ok_dict(self): - return { - "v": 2, - "defined_in": "test_defined_in", - "description": "A version", - "config": {}, - "constraints": [], - "docs": {"show": False}, - "data_tests": [], - "columns": [], - } - - def test_ok(self): - version = self.ContractType( - v=2, - defined_in="test_defined_in", - description="A version", - config={}, - constraints=[], - docs=Docs(show=False), - data_tests=[], - columns=[], - ) - dct = self.get_ok_dict() - self.assert_symmetric(version, dct) - pickle.loads(pickle.dumps(version)) - - def test_bad_version_no_v(self): - version = self.get_ok_dict() - del version["v"] - self.assert_fails_validation(version) - - -@pytest.mark.parametrize( - "left,right,expected_lt", - [ - # same types - (2, 12, True), - (12, 2, False), - ("a", "b", True), - ("b", "a", False), - # mismatched types - numeric - (2, 12.0, True), - (12.0, 2, False), - (2, "12", True), - ("12", 2, False), - # mismatched types - (1, "test", True), - ("test", 1, False), - ], -) -def test_unparsed_version_lt(left, right, expected_lt): - assert (UnparsedVersion(left) < UnparsedVersion(right)) == expected_lt diff --git a/tests/unit/test_deprecations.py b/tests/unit/test_internal_deprecations.py similarity index 100% rename from tests/unit/test_deprecations.py rename to tests/unit/test_internal_deprecations.py From 454de9f5f9b5318796b3e90249c018e328c57936 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Mon, 20 May 2024 16:29:03 -0700 Subject: [PATCH 05/10] more --- .../test_project.py} | 3 +- tests/unit/{ => deps}/test_deps.py | 0 .../test_cli.py} | 0 .../test_docs.py} | 3 +- .../test_docs.py} | 0 tests/unit/test_graph_selection.py | 208 ------------------ 6 files changed, 2 insertions(+), 212 deletions(-) rename tests/unit/{test_contracts_project.py => contracts/test_project.py} (95%) rename tests/unit/{ => deps}/test_deps.py (100%) rename tests/unit/{test_graph_selector_parsing.py => graph/test_cli.py} (100%) rename tests/unit/{test_docs_blocks.py => parser/test_docs.py} (99%) rename tests/unit/{test_docs_generate.py => task/test_docs.py} (100%) delete mode 100644 tests/unit/test_graph_selection.py diff --git a/tests/unit/test_contracts_project.py b/tests/unit/contracts/test_project.py similarity index 95% rename from tests/unit/test_contracts_project.py rename to tests/unit/contracts/test_project.py index b080326fcd2..37e57a33c12 100644 --- a/tests/unit/test_contracts_project.py +++ b/tests/unit/contracts/test_project.py @@ -1,7 +1,6 @@ from dbt.contracts.project import Project from dbt_common.dataclass_schema import ValidationError - -from .utils import ContractTestCase +from tests.unit.utils import ContractTestCase class TestProject(ContractTestCase): diff --git a/tests/unit/test_deps.py b/tests/unit/deps/test_deps.py similarity index 100% rename from tests/unit/test_deps.py rename to tests/unit/deps/test_deps.py diff --git a/tests/unit/test_graph_selector_parsing.py b/tests/unit/graph/test_cli.py similarity index 100% rename from tests/unit/test_graph_selector_parsing.py rename to tests/unit/graph/test_cli.py diff --git a/tests/unit/test_docs_blocks.py b/tests/unit/parser/test_docs.py similarity index 99% rename from tests/unit/test_docs_blocks.py rename to tests/unit/parser/test_docs.py index cb1b5e5e3cc..2eb5d12c383 100644 --- a/tests/unit/test_docs_blocks.py +++ b/tests/unit/parser/test_docs.py @@ -9,8 +9,7 @@ from dbt.node_types import NodeType from dbt.parser import docs from dbt.parser.search import FileBlock - -from .utils import config_from_parts_or_dicts +from tests.unit.utils import config_from_parts_or_dicts set_from_args(Namespace(WARN_ERROR=False), None) diff --git a/tests/unit/test_docs_generate.py b/tests/unit/task/test_docs.py similarity index 100% rename from tests/unit/test_docs_generate.py rename to tests/unit/task/test_docs.py diff --git a/tests/unit/test_graph_selection.py b/tests/unit/test_graph_selection.py deleted file mode 100644 index be283e59926..00000000000 --- a/tests/unit/test_graph_selection.py +++ /dev/null @@ -1,208 +0,0 @@ -import string -from argparse import Namespace -from unittest import mock - -import networkx as nx -import pytest - -import dbt.graph.cli as graph_cli -import dbt.graph.selector as graph_selector -import dbt_common.exceptions -from dbt import flags -from dbt.contracts.project import ProjectFlags -from dbt.node_types import NodeType - -flags.set_from_args(Namespace(), ProjectFlags()) - - -def _get_graph(): - integer_graph = nx.balanced_tree(2, 2, nx.DiGraph()) - - package_mapping = { - i: "m." + ("X" if i % 2 == 0 else "Y") + "." + letter - for (i, letter) in enumerate(string.ascii_lowercase) - } - - # Edges: [(X.a, Y.b), (X.a, X.c), (Y.b, Y.d), (Y.b, X.e), (X.c, Y.f), (X.c, X.g)] - return graph_selector.Graph(nx.relabel_nodes(integer_graph, package_mapping)) - - -def _get_manifest(graph): - nodes = {} - for unique_id in graph: - fqn = unique_id.split(".") - node = mock.MagicMock( - unique_id=unique_id, - fqn=fqn, - package_name=fqn[0], - tags=[], - resource_type=NodeType.Model, - empty=False, - config=mock.MagicMock(enabled=True), - is_versioned=False, - ) - nodes[unique_id] = node - - nodes["m.X.a"].tags = ["abc"] - nodes["m.Y.b"].tags = ["abc", "bcef"] - nodes["m.X.c"].tags = ["abc", "bcef"] - nodes["m.Y.d"].tags = [] - nodes["m.X.e"].tags = ["efg", "bcef"] - nodes["m.Y.f"].tags = ["efg", "bcef"] - nodes["m.X.g"].tags = ["efg"] - return mock.MagicMock(nodes=nodes) - - -@pytest.fixture -def graph(): - return _get_graph() - - -@pytest.fixture -def manifest(graph): - return _get_manifest(graph) - - -def id_macro(arg): - if isinstance(arg, str): - return arg - try: - return "_".join(arg) - except TypeError: - return arg - - -run_specs = [ - # include by fqn - (["X.a"], [], {"m.X.a"}), - # include by tag - (["tag:abc"], [], {"m.X.a", "m.Y.b", "m.X.c"}), - # exclude by tag - (["*"], ["tag:abc"], {"m.Y.d", "m.X.e", "m.Y.f", "m.X.g"}), - # tag + fqn - (["tag:abc", "a"], [], {"m.X.a", "m.Y.b", "m.X.c"}), - (["tag:abc", "d"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.d"}), - # multiple node selection across packages - (["X.a", "b"], [], {"m.X.a", "m.Y.b"}), - (["X.a+"], ["b"], {"m.X.a", "m.X.c", "m.Y.d", "m.X.e", "m.Y.f", "m.X.g"}), - # children - (["X.c+"], [], {"m.X.c", "m.Y.f", "m.X.g"}), - (["X.a+1"], [], {"m.X.a", "m.Y.b", "m.X.c"}), - (["X.a+"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.d"}), - # parents - (["+Y.f"], [], {"m.X.c", "m.Y.f", "m.X.a"}), - (["1+Y.f"], [], {"m.X.c", "m.Y.f"}), - # childrens parents - (["@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), - # multiple selection/exclusion - (["tag:abc", "tag:bcef"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), - (["tag:abc", "tag:bcef"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c"}), - (["tag:abc", "tag:bcef"], ["tag:efg", "a"], {"m.Y.b", "m.X.c"}), - # intersections - (["a,a"], [], {"m.X.a"}), - (["+c,c+"], [], {"m.X.c"}), - (["a,b"], [], set()), - (["tag:abc,tag:bcef"], [], {"m.Y.b", "m.X.c"}), - (["*,tag:abc,a"], [], {"m.X.a"}), - (["a,tag:abc,*"], [], {"m.X.a"}), - (["tag:abc,tag:bcef"], ["c"], {"m.Y.b"}), - (["tag:bcef,tag:efg"], ["tag:bcef,@b"], {"m.Y.f"}), - (["tag:bcef,tag:efg"], ["tag:bcef,@a"], set()), - (["*,@a,+b"], ["*,tag:abc,tag:bcef"], {"m.X.a"}), - (["tag:bcef,tag:efg", "*,tag:abc"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), - (["tag:bcef,tag:efg", "*,tag:abc"], ["e"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.f"}), - (["tag:bcef,tag:efg", "*,tag:abc"], ["e"], {"m.X.a", "m.Y.b", "m.X.c", "m.Y.f"}), - (["tag:bcef,tag:efg", "*,tag:abc"], ["e", "f"], {"m.X.a", "m.Y.b", "m.X.c"}), - (["tag:bcef,tag:efg", "*,tag:abc"], ["tag:abc,tag:bcef"], {"m.X.a", "m.X.e", "m.Y.f"}), - (["tag:bcef,tag:efg", "*,tag:abc"], ["tag:abc,tag:bcef", "tag:abc,a"], {"m.X.e", "m.Y.f"}), -] - - -@pytest.mark.parametrize("include,exclude,expected", run_specs, ids=id_macro) -def test_run_specs(include, exclude, expected, graph, manifest): - selector = graph_selector.NodeSelector(graph, manifest) - spec = graph_cli.parse_difference(include, exclude) - selected, _ = selector.select_nodes(spec) - - assert selected == expected - - -param_specs = [ - ("a", False, None, False, None, "fqn", "a", False), - ("+a", True, None, False, None, "fqn", "a", False), - ("256+a", True, 256, False, None, "fqn", "a", False), - ("a+", False, None, True, None, "fqn", "a", False), - ("a+256", False, None, True, 256, "fqn", "a", False), - ("+a+", True, None, True, None, "fqn", "a", False), - ("16+a+32", True, 16, True, 32, "fqn", "a", False), - ("@a", False, None, False, None, "fqn", "a", True), - ("a.b", False, None, False, None, "fqn", "a.b", False), - ("+a.b", True, None, False, None, "fqn", "a.b", False), - ("256+a.b", True, 256, False, None, "fqn", "a.b", False), - ("a.b+", False, None, True, None, "fqn", "a.b", False), - ("a.b+256", False, None, True, 256, "fqn", "a.b", False), - ("+a.b+", True, None, True, None, "fqn", "a.b", False), - ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False), - ("@a.b", False, None, False, None, "fqn", "a.b", True), - ("a.b.*", False, None, False, None, "fqn", "a.b.*", False), - ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False), - ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False), - ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False), - ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False), - ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False), - ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False), - ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True), - ("tag:a", False, None, False, None, "tag", "a", False), - ("+tag:a", True, None, False, None, "tag", "a", False), - ("256+tag:a", True, 256, False, None, "tag", "a", False), - ("tag:a+", False, None, True, None, "tag", "a", False), - ("tag:a+256", False, None, True, 256, "tag", "a", False), - ("+tag:a+", True, None, True, None, "tag", "a", False), - ("16+tag:a+32", True, 16, True, 32, "tag", "a", False), - ("@tag:a", False, None, False, None, "tag", "a", True), - ("source:a", False, None, False, None, "source", "a", False), - ("source:a+", False, None, True, None, "source", "a", False), - ("source:a+1", False, None, True, 1, "source", "a", False), - ("source:a+32", False, None, True, 32, "source", "a", False), - ("@source:a", False, None, False, None, "source", "a", True), -] - - -@pytest.mark.parametrize( - "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents", - param_specs, - ids=id_macro, -) -def test_parse_specs( - spec, - parents, - parents_depth, - children, - children_depth, - filter_type, - filter_value, - childrens_parents, -): - parsed = graph_selector.SelectionCriteria.from_single_spec(spec) - assert parsed.parents == parents - assert parsed.parents_depth == parents_depth - assert parsed.children == children - assert parsed.children_depth == children_depth - assert parsed.method == filter_type - assert parsed.value == filter_value - assert parsed.childrens_parents == childrens_parents - - -invalid_specs = [ - "@a+", - "@a.b+", - "@a.b*+", - "@tag:a+", - "@source:a+", -] - - -@pytest.mark.parametrize("invalid", invalid_specs, ids=lambda k: str(k)) -def test_invalid_specs(invalid): - with pytest.raises(dbt_common.exceptions.DbtRuntimeError): - graph_selector.SelectionCriteria.from_single_spec(invalid) From 36f20532e59b745eb24e9e46b53abf61398659e9 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Mon, 20 May 2024 22:00:15 -0700 Subject: [PATCH 06/10] more --- .../test_jinja_static.py} | 0 .../test_registry.py} | 0 tests/unit/config/test_selectors.py | 188 +++++- tests/unit/contracts/graph/test_nodes.py | 204 ++++++- .../test_types.py} | 0 tests/unit/graph/test_nodes.py | 197 +++++- tests/unit/graph/test_selector_spec.py | 159 ++++- tests/unit/parser/__init__.py | 0 tests/unit/{ => parser}/test_parser.py | 8 +- tests/unit/parser/test_unit_tests.py | 3 +- tests/unit/test_events.py | 574 ------------------ tests/unit/test_graph_selector_spec.py | 160 ----- tests/unit/test_infer_primary_key.py | 195 ------ tests/unit/test_inject_ctes.py | 198 ------ tests/unit/test_manifest_selectors.py | 201 ------ tests/unit/test_semver.py | 298 --------- 16 files changed, 751 insertions(+), 1634 deletions(-) rename tests/unit/{test_macro_calls.py => clients/test_jinja_static.py} (100%) rename tests/unit/{test_registry_get_request_exception.py => clients/test_registry.py} (100%) rename tests/unit/{test_proto_events.py => events/test_types.py} (100%) create mode 100644 tests/unit/parser/__init__.py rename tests/unit/{ => parser}/test_parser.py (99%) delete mode 100644 tests/unit/test_events.py delete mode 100644 tests/unit/test_graph_selector_spec.py delete mode 100644 tests/unit/test_infer_primary_key.py delete mode 100644 tests/unit/test_inject_ctes.py delete mode 100644 tests/unit/test_manifest_selectors.py delete mode 100644 tests/unit/test_semver.py diff --git a/tests/unit/test_macro_calls.py b/tests/unit/clients/test_jinja_static.py similarity index 100% rename from tests/unit/test_macro_calls.py rename to tests/unit/clients/test_jinja_static.py diff --git a/tests/unit/test_registry_get_request_exception.py b/tests/unit/clients/test_registry.py similarity index 100% rename from tests/unit/test_registry_get_request_exception.py rename to tests/unit/clients/test_registry.py diff --git a/tests/unit/config/test_selectors.py b/tests/unit/config/test_selectors.py index 9688647a00e..d306fb55282 100644 --- a/tests/unit/config/test_selectors.py +++ b/tests/unit/config/test_selectors.py @@ -4,7 +4,8 @@ import yaml import dbt.exceptions -from dbt.config.selectors import SelectorConfig, selector_config_from_data +from dbt.config.selectors import SelectorConfig, SelectorDict, selector_config_from_data +from dbt.exceptions import DbtSelectorsError def get_selector_dict(txt: str) -> dict: @@ -201,3 +202,188 @@ def test_multiple_default_true(self): dbt.exceptions.DbtSelectorsError, "Found multiple selectors with `default: true`:" ): selector_config_from_data(dct) + + def test_compare_cli_non_cli(self): + dct = get_selector_dict( + """\ + selectors: + - name: nightly_diet_snowplow + description: "This uses more CLI-style syntax" + definition: + union: + - intersection: + - '@source:snowplow' + - 'tag:nightly' + - 'models/export' + - exclude: + - intersection: + - 'package:snowplow' + - 'config.materialized:incremental' + - export_performance_timing + - name: nightly_diet_snowplow_full + description: "This is a fuller YAML specification" + definition: + union: + - intersection: + - method: source + value: snowplow + childrens_parents: true + - method: tag + value: nightly + - method: path + value: models/export + - exclude: + - intersection: + - method: package + value: snowplow + - method: config.materialized + value: incremental + - method: fqn + value: export_performance_timing + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + with_strings = sel_dict["nightly_diet_snowplow"]["definition"] + no_strings = sel_dict["nightly_diet_snowplow_full"]["definition"] + self.assertEqual(with_strings, no_strings) + + def test_single_string_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: nightly_selector + definition: + 'tag:nightly' + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "tag", "value": "nightly"} + definition = sel_dict["nightly_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_single_key_value_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: nightly_selector + definition: + tag: nightly + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "tag", "value": "nightly"} + definition = sel_dict["nightly_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_parent_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: kpi_nightly_selector + definition: + '+exposure:kpi_nightly' + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "exposure", "value": "kpi_nightly", "parents": True} + definition = sel_dict["kpi_nightly_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_plus_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: my_model_children_selector + definition: + 'my_model+2' + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + expected = {"method": "fqn", "value": "my_model", "children": True, "children_depth": "2"} + definition = sel_dict["my_model_children_selector"]["definition"] + self.assertEqual(expected, definition) + + def test_selector_definition(self): + dct = get_selector_dict( + """\ + selectors: + - name: default + definition: + union: + - intersection: + - tag: foo + - tag: bar + - name: inherited + definition: + method: selector + value: default + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) + assert sel_dict + definition = sel_dict["default"]["definition"] + expected = sel_dict["inherited"]["definition"] + self.assertEqual(expected, definition) + + def test_selector_definition_with_exclusion(self): + dct = get_selector_dict( + """\ + selectors: + - name: default + definition: + union: + - intersection: + - tag: foo + - tag: bar + - name: inherited + definition: + union: + - method: selector + value: default + - exclude: + - tag: bar + - name: comparison + definition: + union: + - union: + - intersection: + - tag: foo + - tag: bar + - exclude: + - tag: bar + """ + ) + + sel_dict = SelectorDict.parse_from_selectors_list((dct["selectors"])) + assert sel_dict + definition = sel_dict["inherited"]["definition"] + expected = sel_dict["comparison"]["definition"] + self.assertEqual(expected, definition) + + def test_missing_selector(self): + dct = get_selector_dict( + """\ + selectors: + - name: inherited + definition: + method: selector + value: default + """ + ) + with self.assertRaises(DbtSelectorsError) as err: + SelectorDict.parse_from_selectors_list((dct["selectors"])) + + self.assertEqual( + "Existing selector definition for default not found.", str(err.exception.msg) + ) diff --git a/tests/unit/contracts/graph/test_nodes.py b/tests/unit/contracts/graph/test_nodes.py index 67eed2903d2..a498b99dcbc 100644 --- a/tests/unit/contracts/graph/test_nodes.py +++ b/tests/unit/contracts/graph/test_nodes.py @@ -1,11 +1,19 @@ import pickle +import re from dataclasses import replace import pytest from dbt.artifacts.resources import ColumnInfo, TestConfig, TestMetadata +from dbt.compilation import inject_ctes_into_sql from dbt.contracts.files import FileHash -from dbt.contracts.graph.nodes import DependsOn, GenericTestNode, ModelConfig, ModelNode +from dbt.contracts.graph.nodes import ( + DependsOn, + GenericTestNode, + InjectedCTE, + ModelConfig, + ModelNode, +) from dbt.node_types import NodeType from tests.unit.fixtures import generic_test_node, model_node from tests.unit.utils import ( @@ -16,6 +24,12 @@ ) +def norm_whitespace(string): + _RE_COMBINE_WHITESPACE = re.compile(r"\s+") + string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() + return string + + @pytest.fixture def basic_uncompiled_model(): return ModelNode( @@ -602,3 +616,191 @@ def test_compare_to_compiled(basic_uncompiled_schema_test_node, basic_compiled_s compiled, config=fixed_config, unrendered_config=uncompiled.unrendered_config ) assert uncompiled.same_contents(fixed_compiled, "postgres") + + +def test_inject_ctes_simple1(): + starting_sql = "select * from __dbt__cte__base" + ctes = [ + InjectedCTE( + id="model.test.base", + sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", + ) + ] + expected_sql = """with __dbt__cte__base as ( + select * from test16873767336887004702_test_ephemeral.seed + ) select * from __dbt__cte__base""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_simple2(): + starting_sql = "select * from __dbt__cte__ephemeral_level_two" + ctes = [ + InjectedCTE( + id="model.test.ephemeral_level_two", + sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873757769710148165_test_ephemeral"."source_table"\n)', + ) + ] + expected_sql = """with __dbt__cte__ephemeral_level_two as ( + select * from "dbt"."test16873757769710148165_test_ephemeral"."source_table" + ) select * from __dbt__cte__ephemeral_level_two""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_multiple_ctes(): + + starting_sql = "select * from __dbt__cte__ephemeral" + ctes = [ + InjectedCTE( + id="model.test.ephemeral_level_two", + sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873735573223965828_test_ephemeral"."source_table"\n)', + ), + InjectedCTE( + id="model.test.ephemeral", + sql=" __dbt__cte__ephemeral as (\n\nselect * from __dbt__cte__ephemeral_level_two\n)", + ), + ] + expected_sql = """with __dbt__cte__ephemeral_level_two as ( + select * from "dbt"."test16873735573223965828_test_ephemeral"."source_table" + ), __dbt__cte__ephemeral as ( + select * from __dbt__cte__ephemeral_level_two + ) select * from __dbt__cte__ephemeral""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_multiple_ctes_more_complex(): + starting_sql = """select * from __dbt__cte__female_only + union all + select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" + ctes = [ + InjectedCTE( + id="model.test.base", + sql=" __dbt__cte__base as (\n\n\nselect * from test16873757723266827902_test_ephemeral.seed\n)", + ), + InjectedCTE( + id="model.test.base_copy", + sql=" __dbt__cte__base_copy as (\n\n\nselect * from __dbt__cte__base\n)", + ), + InjectedCTE( + id="model.test.female_only", + sql=" __dbt__cte__female_only as (\n\n\nselect * from __dbt__cte__base_copy where gender = 'Female'\n)", + ), + ] + expected_sql = """with __dbt__cte__base as ( + select * from test16873757723266827902_test_ephemeral.seed + ), __dbt__cte__base_copy as ( + select * from __dbt__cte__base + ), __dbt__cte__female_only as ( + select * from __dbt__cte__base_copy where gender = 'Female' + ) select * from __dbt__cte__female_only + union all + select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_starting_with1(): + starting_sql = """ + with internal_cte as (select * from sessions) + select * from internal_cte + """ + ctes = [ + InjectedCTE( + id="cte_id_1", + sql="__dbt__cte__ephemeral as (select * from table)", + ), + InjectedCTE( + id="cte_id_2", + sql="__dbt__cte__events as (select id, type from events)", + ), + ] + expected_sql = """with __dbt__cte__ephemeral as (select * from table), + __dbt__cte__events as (select id, type from events), + internal_cte as (select * from sessions) + select * from internal_cte""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_starting_with2(): + starting_sql = """with my_other_cool_cte as ( + select id, name from __dbt__cte__ephemeral + where id > 1000 + ) + select name, id from my_other_cool_cte""" + ctes = [ + InjectedCTE( + id="model.singular_tests_ephemeral.ephemeral", + sql=' __dbt__cte__ephemeral as (\n\n\nwith my_cool_cte as (\n select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base"\n)\nselect id, name from my_cool_cte where id is not null\n)', + ) + ] + expected_sql = """with __dbt__cte__ephemeral as ( + with my_cool_cte as ( + select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base" + ) + select id, name from my_cool_cte where id is not null + ), my_other_cool_cte as ( + select id, name from __dbt__cte__ephemeral + where id > 1000 + ) + select name, id from my_other_cool_cte""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_comment_with(): + # Test injection with a comment containing "with" + starting_sql = """ + --- This is sql with a comment + select * from __dbt__cte__base + """ + ctes = [ + InjectedCTE( + id="model.test.base", + sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", + ) + ] + expected_sql = """with __dbt__cte__base as ( + select * from test16873767336887004702_test_ephemeral.seed + ) --- This is sql with a comment + select * from __dbt__cte__base""" + + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) + + +def test_inject_ctes_with_recursive(): + # Test injection with "recursive" keyword + starting_sql = """ + with recursive t(n) as ( + select * from __dbt__cte__first_ephemeral_model + union all + select n+1 from t where n < 100 + ) + select sum(n) from t + """ + ctes = [ + InjectedCTE( + id="model.test.first_ephemeral_model", + sql=" __dbt__cte__first_ephemeral_model as (\n\nselect 1 as fun\n)", + ) + ] + expected_sql = """with recursive __dbt__cte__first_ephemeral_model as ( + select 1 as fun + ), t(n) as ( + select * from __dbt__cte__first_ephemeral_model + union all + select n+1 from t where n < 100 + ) + select sum(n) from t + """ + generated_sql = inject_ctes_into_sql(starting_sql, ctes) + assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) diff --git a/tests/unit/test_proto_events.py b/tests/unit/events/test_types.py similarity index 100% rename from tests/unit/test_proto_events.py rename to tests/unit/events/test_types.py diff --git a/tests/unit/graph/test_nodes.py b/tests/unit/graph/test_nodes.py index 897eee5111c..4a62db34b11 100644 --- a/tests/unit/graph/test_nodes.py +++ b/tests/unit/graph/test_nodes.py @@ -9,10 +9,17 @@ EntityType, ) -from dbt.artifacts.resources import Defaults, Dimension, Entity, Measure +from dbt.artifacts.resources import Defaults, Dimension, Entity, Measure, TestMetadata from dbt.artifacts.resources.v1.semantic_model import NodeRelation -from dbt.contracts.graph.nodes import SemanticModel +from dbt.contracts.graph.model_config import TestConfig +from dbt.contracts.graph.nodes import ColumnInfo, SemanticModel from dbt.node_types import NodeType +from dbt_common.contracts.constraints import ( + ColumnLevelConstraint, + ConstraintType, + ModelLevelConstraint, +) +from tests.unit.fixtures import generic_test_node, model_node class TestSemanticModel: @@ -109,3 +116,189 @@ def test_semantic_model_same_contents_different_node_relation( default_semantic_model_copy.node_relation.alias = "test_another_alias" # Relation should not be consided in same_contents assert default_semantic_model.same_contents(default_semantic_model_copy) + + +# Infer primary key +def test_no_primary_key(): + model = model_node() + assert model.infer_primary_key([]) == [] + + +def test_primary_key_model_constraint(): + model = model_node() + model.constraints = [ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk"])] + assertSameContents(model.infer_primary_key([]), ["pk"]) + + model.constraints = [ + ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk1", "pk2"]) + ] + assertSameContents(model.infer_primary_key([]), ["pk1", "pk2"]) + + +def test_primary_key_column_constraint(): + model = model_node() + model.columns = { + "column1": ColumnInfo( + "column1", constraints=[ColumnLevelConstraint(type=ConstraintType.primary_key)] + ), + "column2": ColumnInfo("column2"), + } + assertSameContents(model.infer_primary_key([]), ["column1"]) + + +def test_unique_non_null_single(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column1"}) + test3 = generic_test_node() + test3.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2"}) + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1"]) + + +def test_unique_non_null_multiple(): + model = model_node() + tests = [] + for i in range(2): + for enabled in [True, False]: + test1 = generic_test_node() + test1.test_metadata = TestMetadata( + name="unique", kwargs={"column_name": "column" + str(i) + str(enabled)} + ) + test1.config = TestConfig(enabled=enabled) + test2 = generic_test_node() + test2.test_metadata = TestMetadata( + name="not_null", kwargs={"column_name": "column" + str(i) + str(enabled)} + ) + test2.config = TestConfig(enabled=enabled) + tests.extend([test1, test2]) + + assertSameContents( + model.infer_primary_key(tests), + ["column0True", "column1True", "column0False", "column1False"], + ) + + +def test_enabled_unique_single(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column3"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1"]) + + +def test_enabled_unique_multiple(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) + + +def test_enabled_unique_combo_single(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata( + name="unique_combination_of_columns", + kwargs={"combination_of_columns": ["column1", "column2"]}, + ) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata( + name="unique_combination_of_columns", + kwargs={"combination_of_columns": ["column3", "column4"]}, + ) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) + + +def test_enabled_unique_combo_multiple(): + model = model_node() + test1 = generic_test_node() + test1.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} + ) + test2 = generic_test_node() + test2.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} + ) + + tests = [test1, test2] + assertSameContents( + model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] + ) + + +def test_disabled_unique_single(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column2"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1"]) + + +def test_disabled_unique_multiple(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) + + +def test_disabled_unique_combo_single(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} + ) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata( + name="random", kwargs={"combination_of_columns": ["column3", "column4"]} + ) + + tests = [test1, test2] + assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) + + +def test_disabled_unique_combo_multiple(): + model = model_node() + test1 = generic_test_node() + test1.config = TestConfig(enabled=False) + test1.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} + ) + test2 = generic_test_node() + test2.config = TestConfig(enabled=False) + test2.test_metadata = TestMetadata( + name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} + ) + + tests = [test1, test2] + assertSameContents( + model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] + ) + + +def assertSameContents(list1, list2): + assert sorted(list1) == sorted(list2) diff --git a/tests/unit/graph/test_selector_spec.py b/tests/unit/graph/test_selector_spec.py index 265e387bc0a..451b107d85c 100644 --- a/tests/unit/graph/test_selector_spec.py +++ b/tests/unit/graph/test_selector_spec.py @@ -1,8 +1,17 @@ +import os from unittest.mock import patch import pytest -from dbt.graph.selector_spec import IndirectSelection, SelectionCriteria +from dbt.exceptions import DbtRuntimeError +from dbt.graph.selector_methods import MethodName +from dbt.graph.selector_spec import ( + IndirectSelection, + SelectionCriteria, + SelectionDifference, + SelectionIntersection, + SelectionUnion, +) @pytest.mark.parametrize( @@ -47,3 +56,151 @@ def test_selection_criteria_default_indirect_value(indirect_selection_value, exp assert ( selection_criteria_with_indirect_selection_specified.indirect_selection == "buildable" ) + + +def test_raw_parse_simple(): + raw = "asdf" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == raw + assert not result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_path(): + raw = os.path.join("asdf", "*") + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.Path + assert result.method_arguments == [] + assert result.value == raw + assert not result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_path_modified(): + raw = "@" + os.path.join("asdf", "*") + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.Path + assert result.method_arguments == [] + assert result.value == raw[1:] + assert result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_fqn_parents(): + raw = "+asdf" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "asdf" + assert not result.childrens_parents + assert not result.children + assert result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_simple_infer_fqn_children(): + raw = "asdf+" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "asdf" + assert not result.childrens_parents + assert result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_complex(): + raw = "2+config.arg.secondarg:argument_value+4" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.Config + assert result.method_arguments == ["arg", "secondarg"] + assert result.value == "argument_value" + assert not result.childrens_parents + assert result.children + assert result.parents + assert result.parents_depth == 2 + assert result.children_depth == 4 + + +def test_raw_parse_weird(): + # you can have an empty method name (defaults to FQN/path) and you can have + # an empty value, so you can also have this... + result = SelectionCriteria.from_single_spec("") + assert result.raw == "" + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "" + assert not result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + + +def test_raw_parse_invalid(): + with pytest.raises(DbtRuntimeError): + SelectionCriteria.from_single_spec("invalid_method:something") + + with pytest.raises(DbtRuntimeError): + SelectionCriteria.from_single_spec("@foo+") + + +def test_intersection(): + fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") + fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") + intersection = SelectionIntersection(components=[fqn_a, fqn_b]) + assert list(intersection) == [fqn_a, fqn_b] + combined = intersection.combine_selections( + [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] + ) + assert combined == {"model_c"} + + +def test_difference(): + fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") + fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") + difference = SelectionDifference(components=[fqn_a, fqn_b]) + assert list(difference) == [fqn_a, fqn_b] + combined = difference.combine_selections( + [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] + ) + assert combined == {"model_a", "model_b"} + + fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") + difference = SelectionDifference(components=[fqn_a, fqn_b, fqn_c]) + assert list(difference) == [fqn_a, fqn_b, fqn_c] + combined = difference.combine_selections( + [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}, {"model_a"}] + ) + assert combined == {"model_b"} + + +def test_union(): + fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") + fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") + fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") + difference = SelectionUnion(components=[fqn_a, fqn_b, fqn_c]) + combined = difference.combine_selections( + [{"model_a", "model_b"}, {"model_b", "model_c"}, {"model_d"}] + ) + assert combined == {"model_a", "model_b", "model_c", "model_d"} diff --git a/tests/unit/parser/__init__.py b/tests/unit/parser/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_parser.py b/tests/unit/parser/test_parser.py similarity index 99% rename from tests/unit/test_parser.py rename to tests/unit/parser/test_parser.py index c57dd306dcd..42398f48f39 100644 --- a/tests/unit/test_parser.py +++ b/tests/unit/parser/test_parser.py @@ -53,8 +53,12 @@ ) from dbt.parser.search import FileBlock from dbt.parser.sources import SourcePatcher - -from .utils import MockNode, config_from_parts_or_dicts, generate_name_macros, normalize +from tests.unit.utils import ( + MockNode, + config_from_parts_or_dicts, + generate_name_macros, + normalize, +) set_from_args(Namespace(WARN_ERROR=False), None) diff --git a/tests/unit/parser/test_unit_tests.py b/tests/unit/parser/test_unit_tests.py index 39832fa89f9..15f018f6446 100644 --- a/tests/unit/parser/test_unit_tests.py +++ b/tests/unit/parser/test_unit_tests.py @@ -5,9 +5,10 @@ from dbt.contracts.graph.unparsed import UnitTestOutputFixture from dbt.parser import SchemaParser from dbt.parser.unit_tests import UnitTestParser -from tests.unit.test_parser import SchemaParserTest, assertEqualNodes from tests.unit.utils import MockNode +from .test_parser import SchemaParserTest, assertEqualNodes + UNIT_TEST_MODEL_NOT_FOUND_SOURCE = """ unit_tests: - name: test_my_model_doesnt_exist diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py deleted file mode 100644 index bd9892bcc6c..00000000000 --- a/tests/unit/test_events.py +++ /dev/null @@ -1,574 +0,0 @@ -import logging -import re -from argparse import Namespace -from typing import TypeVar - -import pytest - -from dbt.adapters.events import types as adapter_types -from dbt.adapters.events.logging import AdapterLogger -from dbt.artifacts.schemas.results import RunStatus, TimingInfo -from dbt.artifacts.schemas.run import RunResult -from dbt.events import types as core_types -from dbt.events.base_types import ( - CoreBaseEvent, - DebugLevel, - DynamicLevel, - ErrorLevel, - InfoLevel, - TestLevel, - WarnLevel, -) -from dbt.events.types import RunResultError -from dbt.flags import set_from_args -from dbt.task.printer import print_run_result_error -from dbt_common.events import types -from dbt_common.events.base_types import msg_from_base_event -from dbt_common.events.event_manager import EventManager, TestEventManager -from dbt_common.events.event_manager_client import ctx_set_event_manager -from dbt_common.events.functions import msg_to_dict, msg_to_json -from dbt_common.events.helpers import get_json_string_utcnow - -set_from_args(Namespace(WARN_ERROR=False), None) - - -# takes in a class and finds any subclasses for it -def get_all_subclasses(cls): - all_subclasses = [] - for subclass in cls.__subclasses__(): - if subclass not in [TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel, DynamicLevel]: - all_subclasses.append(subclass) - all_subclasses.extend(get_all_subclasses(subclass)) - return set(all_subclasses) - - -class TestAdapterLogger: - # this interface is documented for adapter maintainers to plug into - # so we should test that it at the very least doesn't explode. - def test_basic_adapter_logging_interface(self): - logger = AdapterLogger("dbt_tests") - logger.debug("debug message") - logger.info("info message") - logger.warning("warning message") - logger.error("error message") - logger.exception("exception message") - logger.critical("exception message") - - # python loggers allow deferring string formatting via this signature: - def test_formatting(self): - logger = AdapterLogger("dbt_tests") - # tests that it doesn't throw - logger.debug("hello {}", "world") - - # enters lower in the call stack to test that it formats correctly - event = adapter_types.AdapterEventDebug( - name="dbt_tests", base_msg="hello {}", args=["world"] - ) - assert "hello world" in event.message() - - # tests that it doesn't throw - logger.debug("1 2 {}", "3") - - # enters lower in the call stack to test that it formats correctly - event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg="1 2 {}", args=[3]) - assert "1 2 3" in event.message() - - # tests that it doesn't throw - logger.debug("boop{x}boop") - - # enters lower in the call stack to test that it formats correctly - # in this case it's that we didn't attempt to replace anything since there - # were no args passed after the initial message - event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg="boop{x}boop", args=[]) - assert "boop{x}boop" in event.message() - - # ensure AdapterLogger and subclasses makes all base_msg members - # of type string; when someone writes logger.debug(a) where a is - # any non-string object - event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg=[1, 2, 3], args=[3]) - assert isinstance(event.base_msg, str) - - event = core_types.JinjaLogDebug(msg=[1, 2, 3]) - assert isinstance(event.msg, str) - - def test_set_adapter_dependency_log_level(self): - logger = AdapterLogger("dbt_tests") - package_log = logging.getLogger("test_package_log") - logger.set_adapter_dependency_log_level("test_package_log", "DEBUG") - package_log.debug("debug message") - - -class TestEventCodes: - - # checks to see if event codes are duplicated to keep codes singluar and clear. - # also checks that event codes follow correct namming convention ex. E001 - def test_event_codes(self): - all_concrete = get_all_subclasses(CoreBaseEvent) - all_codes = set() - - for event_cls in all_concrete: - code = event_cls.code(event_cls) - # must be in the form 1 capital letter, 3 digits - assert re.match("^[A-Z][0-9]{3}", code) - # cannot have been used already - assert ( - code not in all_codes - ), f"{code} is assigned more than once. Check types.py for duplicates." - all_codes.add(code) - - -sample_values = [ - # N.B. Events instantiated here include the module prefix in order to - # avoid having the entire list twice in the code. - # A - pre-project loading - core_types.MainReportVersion(version=""), - core_types.MainReportArgs(args={}), - core_types.MainTrackingUserState(user_state=""), - core_types.MissingProfileTarget(profile_name="", target_name=""), - core_types.InvalidOptionYAML(option_name="vars"), - core_types.LogDbtProjectError(), - core_types.LogDbtProfileError(), - core_types.StarterProjectPath(dir=""), - core_types.ConfigFolderDirectory(dir=""), - core_types.NoSampleProfileFound(adapter=""), - core_types.ProfileWrittenWithSample(name="", path=""), - core_types.ProfileWrittenWithTargetTemplateYAML(name="", path=""), - core_types.ProfileWrittenWithProjectTemplateYAML(name="", path=""), - core_types.SettingUpProfile(), - core_types.InvalidProfileTemplateYAML(), - core_types.ProjectNameAlreadyExists(name=""), - core_types.ProjectCreated(project_name=""), - # D - Deprecations ====================== - core_types.PackageRedirectDeprecation(old_name="", new_name=""), - core_types.PackageInstallPathDeprecation(), - core_types.ConfigSourcePathDeprecation(deprecated_path="", exp_path=""), - core_types.ConfigDataPathDeprecation(deprecated_path="", exp_path=""), - adapter_types.AdapterDeprecationWarning(old_name="", new_name=""), - core_types.MetricAttributesRenamed(metric_name=""), - core_types.ExposureNameDeprecation(exposure=""), - core_types.InternalDeprecation(name="", reason="", suggested_action="", version=""), - core_types.EnvironmentVariableRenamed(old_name="", new_name=""), - core_types.ConfigLogPathDeprecation(deprecated_path=""), - core_types.ConfigTargetPathDeprecation(deprecated_path=""), - adapter_types.CollectFreshnessReturnSignature(), - core_types.TestsConfigDeprecation(deprecated_path="", exp_path=""), - core_types.ProjectFlagsMovedDeprecation(), - core_types.SpacesInResourceNameDeprecation(unique_id="", level=""), - core_types.ResourceNamesWithSpacesDeprecation( - count_invalid_names=1, show_debug_hint=True, level="" - ), - core_types.PackageMaterializationOverrideDeprecation( - package_name="my_package", materialization_name="view" - ), - core_types.SourceFreshnessProjectHooksNotRun(), - # E - DB Adapter ====================== - adapter_types.AdapterEventDebug(), - adapter_types.AdapterEventInfo(), - adapter_types.AdapterEventWarning(), - adapter_types.AdapterEventError(), - adapter_types.AdapterRegistered(adapter_name="dbt-awesome", adapter_version="1.2.3"), - adapter_types.NewConnection(conn_type="", conn_name=""), - adapter_types.ConnectionReused(conn_name=""), - adapter_types.ConnectionLeftOpenInCleanup(conn_name=""), - adapter_types.ConnectionClosedInCleanup(conn_name=""), - adapter_types.RollbackFailed(conn_name=""), - adapter_types.ConnectionClosed(conn_name=""), - adapter_types.ConnectionLeftOpen(conn_name=""), - adapter_types.Rollback(conn_name=""), - adapter_types.CacheMiss(conn_name="", database="", schema=""), - adapter_types.ListRelations(database="", schema=""), - adapter_types.ConnectionUsed(conn_type="", conn_name=""), - adapter_types.SQLQuery(conn_name="", sql=""), - adapter_types.SQLQueryStatus(status="", elapsed=0.1), - adapter_types.SQLCommit(conn_name=""), - adapter_types.ColTypeChange( - orig_type="", - new_type="", - table={"database": "", "schema": "", "identifier": ""}, - ), - adapter_types.SchemaCreation(relation={"database": "", "schema": "", "identifier": ""}), - adapter_types.SchemaDrop(relation={"database": "", "schema": "", "identifier": ""}), - adapter_types.CacheAction( - action="adding_relation", - ref_key={"database": "", "schema": "", "identifier": ""}, - ref_key_2={"database": "", "schema": "", "identifier": ""}, - ), - adapter_types.CacheDumpGraph(before_after="before", action="rename", dump=dict()), - adapter_types.AdapterImportError(exc=""), - adapter_types.PluginLoadError(exc_info=""), - adapter_types.NewConnectionOpening(connection_state=""), - adapter_types.CodeExecution(conn_name="", code_content=""), - adapter_types.CodeExecutionStatus(status="", elapsed=0.1), - adapter_types.CatalogGenerationError(exc=""), - adapter_types.WriteCatalogFailure(num_exceptions=0), - adapter_types.CatalogWritten(path=""), - adapter_types.CannotGenerateDocs(), - adapter_types.BuildingCatalog(), - adapter_types.DatabaseErrorRunningHook(hook_type=""), - adapter_types.HooksRunning(num_hooks=0, hook_type=""), - adapter_types.FinishedRunningStats(stat_line="", execution="", execution_time=0), - adapter_types.ConstraintNotEnforced(constraint="", adapter=""), - adapter_types.ConstraintNotSupported(constraint="", adapter=""), - # I - Project parsing ====================== - core_types.InputFileDiffError(category="testing", file_id="my_file"), - core_types.InvalidValueForField(field_name="test", field_value="test"), - core_types.ValidationWarning(resource_type="model", field_name="access", node_name="my_macro"), - core_types.ParsePerfInfoPath(path=""), - core_types.PartialParsingErrorProcessingFile(file=""), - core_types.PartialParsingFile(file_id=""), - core_types.PartialParsingError(exc_info={}), - core_types.PartialParsingSkipParsing(), - core_types.UnableToPartialParse(reason="something went wrong"), - core_types.StateCheckVarsHash(vars="testing", target="testing", profile="testing"), - core_types.PartialParsingNotEnabled(), - core_types.ParsedFileLoadFailed(path="", exc="", exc_info=""), - core_types.PartialParsingEnabled(deleted=0, added=0, changed=0), - core_types.PartialParsingFile(file_id=""), - core_types.InvalidDisabledTargetInTestNode( - resource_type_title="", - unique_id="", - original_file_path="", - target_kind="", - target_name="", - target_package="", - ), - core_types.UnusedResourceConfigPath(unused_config_paths=[]), - core_types.SeedIncreased(package_name="", name=""), - core_types.SeedExceedsLimitSamePath(package_name="", name=""), - core_types.SeedExceedsLimitAndPathChanged(package_name="", name=""), - core_types.SeedExceedsLimitChecksumChanged(package_name="", name="", checksum_name=""), - core_types.UnusedTables(unused_tables=[]), - core_types.WrongResourceSchemaFile( - patch_name="", resource_type="", file_path="", plural_resource_type="" - ), - core_types.NoNodeForYamlKey(patch_name="", yaml_key="", file_path=""), - core_types.MacroNotFoundForPatch(patch_name=""), - core_types.NodeNotFoundOrDisabled( - original_file_path="", - unique_id="", - resource_type_title="", - target_name="", - target_kind="", - target_package="", - disabled="", - ), - core_types.JinjaLogWarning(), - core_types.JinjaLogInfo(msg=""), - core_types.JinjaLogDebug(msg=""), - core_types.UnpinnedRefNewVersionAvailable( - ref_node_name="", ref_node_package="", ref_node_version="", ref_max_version="" - ), - core_types.DeprecatedModel(model_name="", model_version="", deprecation_date=""), - core_types.DeprecatedReference( - model_name="", - ref_model_name="", - ref_model_package="", - ref_model_deprecation_date="", - ref_model_latest_version="", - ), - core_types.UpcomingReferenceDeprecation( - model_name="", - ref_model_name="", - ref_model_package="", - ref_model_deprecation_date="", - ref_model_latest_version="", - ), - core_types.UnsupportedConstraintMaterialization(materialized=""), - core_types.ParseInlineNodeError(exc=""), - core_types.SemanticValidationFailure(msg=""), - core_types.UnversionedBreakingChange( - breaking_changes=[], - model_name="", - model_file_path="", - contract_enforced_disabled=True, - columns_removed=[], - column_type_changes=[], - enforced_column_constraint_removed=[], - enforced_model_constraint_removed=[], - materialization_changed=[], - ), - core_types.WarnStateTargetEqual(state_path=""), - core_types.FreshnessConfigProblem(msg=""), - core_types.SemanticValidationFailure(msg=""), - # M - Deps generation ====================== - core_types.GitSparseCheckoutSubdirectory(subdir=""), - core_types.GitProgressCheckoutRevision(revision=""), - core_types.GitProgressUpdatingExistingDependency(dir=""), - core_types.GitProgressPullingNewDependency(dir=""), - core_types.GitNothingToDo(sha=""), - core_types.GitProgressUpdatedCheckoutRange(start_sha="", end_sha=""), - core_types.GitProgressCheckedOutAt(end_sha=""), - core_types.RegistryProgressGETRequest(url=""), - core_types.RegistryProgressGETResponse(url="", resp_code=1234), - core_types.SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec=""), - core_types.DepsNoPackagesFound(), - core_types.DepsStartPackageInstall(package_name=""), - core_types.DepsInstallInfo(version_name=""), - core_types.DepsUpdateAvailable(version_latest=""), - core_types.DepsUpToDate(), - core_types.DepsListSubdirectory(subdirectory=""), - core_types.DepsNotifyUpdatesAvailable(packages=["my_pkg", "other_pkg"]), - types.RetryExternalCall(attempt=0, max=0), - types.RecordRetryException(exc=""), - core_types.RegistryIndexProgressGETRequest(url=""), - core_types.RegistryIndexProgressGETResponse(url="", resp_code=1234), - core_types.RegistryResponseUnexpectedType(response=""), - core_types.RegistryResponseMissingTopKeys(response=""), - core_types.RegistryResponseMissingNestedKeys(response=""), - core_types.RegistryResponseExtraNestedKeys(response=""), - core_types.DepsSetDownloadDirectory(path=""), - core_types.DepsLockUpdating(lock_filepath=""), - core_types.DepsAddPackage(package_name="", version="", packages_filepath=""), - core_types.DepsFoundDuplicatePackage(removed_package={}), - core_types.DepsScrubbedPackageName(package_name=""), - core_types.DepsUnpinned(revision="", git=""), - core_types.NoNodesForSelectionCriteria(spec_raw=""), - # Q - Node execution ====================== - core_types.RunningOperationCaughtError(exc=""), - core_types.CompileComplete(), - core_types.FreshnessCheckComplete(), - core_types.SeedHeader(header=""), - core_types.SQLRunnerException(exc=""), - core_types.LogTestResult( - name="", - index=0, - num_models=0, - execution_time=0, - num_failures=0, - ), - core_types.LogStartLine(description="", index=0, total=0), - core_types.LogModelResult( - description="", - status="", - index=0, - total=0, - execution_time=0, - ), - core_types.LogSnapshotResult( - status="", - description="", - cfg={}, - index=0, - total=0, - execution_time=0, - ), - core_types.LogSeedResult( - status="", - index=0, - total=0, - execution_time=0, - schema="", - relation="", - ), - core_types.LogFreshnessResult( - source_name="", - table_name="", - index=0, - total=0, - execution_time=0, - ), - core_types.LogNodeNoOpResult( - description="", - status="", - index=0, - total=0, - execution_time=0, - ), - core_types.LogCancelLine(conn_name=""), - core_types.DefaultSelector(name=""), - core_types.NodeStart(), - core_types.NodeFinished(), - core_types.QueryCancelationUnsupported(type=""), - core_types.ConcurrencyLine(num_threads=0, target_name=""), - core_types.WritingInjectedSQLForNode(), - core_types.NodeCompiling(), - core_types.NodeExecuting(), - core_types.LogHookStartLine( - statement="", - index=0, - total=0, - ), - core_types.LogHookEndLine( - statement="", - status="", - index=0, - total=0, - execution_time=0, - ), - core_types.SkippingDetails( - resource_type="", - schema="", - node_name="", - index=0, - total=0, - ), - core_types.NothingToDo(), - core_types.RunningOperationUncaughtError(exc=""), - core_types.EndRunResult(), - core_types.NoNodesSelected(), - core_types.CommandCompleted( - command="", - success=True, - elapsed=0.1, - completed_at=get_json_string_utcnow(), - ), - core_types.ShowNode(node_name="", preview="", is_inline=True, unique_id="model.test.my_model"), - core_types.CompiledNode( - node_name="", compiled="", is_inline=True, unique_id="model.test.my_model" - ), - # W - Node testing ====================== - core_types.CatchableExceptionOnRun(exc=""), - core_types.InternalErrorOnRun(build_path="", exc=""), - core_types.GenericExceptionOnRun(build_path="", unique_id="", exc=""), - core_types.NodeConnectionReleaseError(node_name="", exc=""), - core_types.FoundStats(stat_line=""), - # Z - misc ====================== - core_types.MainKeyboardInterrupt(), - core_types.MainEncounteredError(exc=""), - core_types.MainStackTrace(stack_trace=""), - types.SystemCouldNotWrite(path="", reason="", exc=""), - types.SystemExecutingCmd(cmd=[""]), - types.SystemStdOut(bmsg=str(b"")), - types.SystemStdErr(bmsg=str(b"")), - types.SystemReportReturnCode(returncode=0), - core_types.TimingInfoCollected(), - core_types.LogDebugStackTrace(), - core_types.CheckCleanPath(path=""), - core_types.ConfirmCleanPath(path=""), - core_types.ProtectedCleanPath(path=""), - core_types.FinishedCleanPaths(), - core_types.OpenCommand(open_cmd="", profiles_dir=""), - core_types.RunResultWarning(resource_type="", node_name="", path=""), - core_types.RunResultFailure(resource_type="", node_name="", path=""), - core_types.StatsLine(stats={"error": 0, "skip": 0, "pass": 0, "warn": 0, "total": 0}), - core_types.RunResultError(msg=""), - core_types.RunResultErrorNoMessage(status=""), - core_types.SQLCompiledPath(path=""), - core_types.CheckNodeTestFailure(relation_name=""), - core_types.EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False), - core_types.LogSkipBecauseError(schema="", relation="", index=0, total=0), - core_types.EnsureGitInstalled(), - core_types.DepsCreatingLocalSymlink(), - core_types.DepsSymlinkNotAvailable(), - core_types.DisableTracking(), - core_types.SendingEvent(kwargs=""), - core_types.SendEventFailure(), - core_types.FlushEvents(), - core_types.FlushEventsFailure(), - types.Formatting(), - core_types.TrackingInitializeFailure(), - core_types.RunResultWarningMessage(), - core_types.DebugCmdOut(), - core_types.DebugCmdResult(), - core_types.ListCmdOut(), - types.Note(msg="This is a note."), - core_types.ResourceReport(), -] - - -class TestEventJSONSerialization: - - # attempts to test that every event is serializable to json. - # event types that take `Any` are not possible to test in this way since some will serialize - # just fine and others won't. - def test_all_serializable(self): - all_non_abstract_events = set( - get_all_subclasses(CoreBaseEvent), - ) - all_event_values_list = list(map(lambda x: x.__class__, sample_values)) - diff = all_non_abstract_events.difference(set(all_event_values_list)) - assert ( - not diff - ), f"{diff}test is missing concrete values in `sample_values`. Please add the values for the aforementioned event classes" - - # make sure everything in the list is a value not a type - for event in sample_values: - assert type(event) != type - - # if we have everything we need to test, try to serialize everything - count = 0 - for event in sample_values: - msg = msg_from_base_event(event) - print(f"--- msg: {msg.info.name}") - # Serialize to dictionary - try: - msg_to_dict(msg) - except Exception as e: - raise Exception( - f"{event} can not be converted to a dict. Originating exception: {e}" - ) - # Serialize to json - try: - msg_to_json(msg) - except Exception as e: - raise Exception(f"{event} is not serializable to json. Originating exception: {e}") - # Serialize to binary - try: - msg.SerializeToString() - except Exception as e: - raise Exception( - f"{event} is not serializable to binary protobuf. Originating exception: {e}" - ) - count += 1 - print(f"--- Found {count} events") - - -T = TypeVar("T") - - -def test_date_serialization(): - ti = TimingInfo("test") - ti.begin() - ti.end() - ti_dict = ti.to_dict() - assert ti_dict["started_at"].endswith("Z") - assert ti_dict["completed_at"].endswith("Z") - - -def test_bad_serialization(): - """Tests that bad serialization enters the proper exception handling - - When pytest is in use the exception handling of `BaseEvent` raises an - exception. When pytest isn't present, it fires a Note event. Thus to test - that bad serializations are properly handled, the best we can do is test - that the exception handling path is used. - """ - - with pytest.raises(Exception) as excinfo: - types.Note(param_event_doesnt_have="This should break") - - assert ( - str(excinfo.value) - == "[Note]: Unable to parse dict {'param_event_doesnt_have': 'This should break'}" - ) - - -def test_single_run_error(): - - try: - # Add a recording event manager to the context, so we can test events. - event_mgr = TestEventManager() - ctx_set_event_manager(event_mgr) - - error_result = RunResult( - status=RunStatus.Error, - timing=[], - thread_id="", - execution_time=0.0, - node=None, - adapter_response=dict(), - message="oh no!", - failures=[], - ) - - print_run_result_error(error_result) - events = [e for e in event_mgr.event_history if isinstance(e[0], RunResultError)] - - assert len(events) == 1 - assert events[0][0].msg == "oh no!" - - finally: - # Set an empty event manager unconditionally on exit. This is an early - # attempt at unit testing events, and we need to think about how it - # could be done in a thread safe way in the long run. - ctx_set_event_manager(EventManager()) diff --git a/tests/unit/test_graph_selector_spec.py b/tests/unit/test_graph_selector_spec.py deleted file mode 100644 index 6164f113158..00000000000 --- a/tests/unit/test_graph_selector_spec.py +++ /dev/null @@ -1,160 +0,0 @@ -import os - -import pytest - -from dbt.exceptions import DbtRuntimeError -from dbt.graph.selector_methods import MethodName -from dbt.graph.selector_spec import ( - SelectionCriteria, - SelectionDifference, - SelectionIntersection, - SelectionUnion, -) - - -def test_raw_parse_simple(): - raw = "asdf" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == raw - assert not result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_path(): - raw = os.path.join("asdf", "*") - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.Path - assert result.method_arguments == [] - assert result.value == raw - assert not result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_path_modified(): - raw = "@" + os.path.join("asdf", "*") - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.Path - assert result.method_arguments == [] - assert result.value == raw[1:] - assert result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_fqn_parents(): - raw = "+asdf" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == "asdf" - assert not result.childrens_parents - assert not result.children - assert result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_simple_infer_fqn_children(): - raw = "asdf+" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == "asdf" - assert not result.childrens_parents - assert result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_complex(): - raw = "2+config.arg.secondarg:argument_value+4" - result = SelectionCriteria.from_single_spec(raw) - assert result.raw == raw - assert result.method == MethodName.Config - assert result.method_arguments == ["arg", "secondarg"] - assert result.value == "argument_value" - assert not result.childrens_parents - assert result.children - assert result.parents - assert result.parents_depth == 2 - assert result.children_depth == 4 - - -def test_raw_parse_weird(): - # you can have an empty method name (defaults to FQN/path) and you can have - # an empty value, so you can also have this... - result = SelectionCriteria.from_single_spec("") - assert result.raw == "" - assert result.method == MethodName.FQN - assert result.method_arguments == [] - assert result.value == "" - assert not result.childrens_parents - assert not result.children - assert not result.parents - assert result.parents_depth is None - assert result.children_depth is None - - -def test_raw_parse_invalid(): - with pytest.raises(DbtRuntimeError): - SelectionCriteria.from_single_spec("invalid_method:something") - - with pytest.raises(DbtRuntimeError): - SelectionCriteria.from_single_spec("@foo+") - - -def test_intersection(): - fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") - fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") - intersection = SelectionIntersection(components=[fqn_a, fqn_b]) - assert list(intersection) == [fqn_a, fqn_b] - combined = intersection.combine_selections( - [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] - ) - assert combined == {"model_c"} - - -def test_difference(): - fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") - fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") - difference = SelectionDifference(components=[fqn_a, fqn_b]) - assert list(difference) == [fqn_a, fqn_b] - combined = difference.combine_selections( - [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}] - ) - assert combined == {"model_a", "model_b"} - - fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") - difference = SelectionDifference(components=[fqn_a, fqn_b, fqn_c]) - assert list(difference) == [fqn_a, fqn_b, fqn_c] - combined = difference.combine_selections( - [{"model_a", "model_b", "model_c"}, {"model_c", "model_d"}, {"model_a"}] - ) - assert combined == {"model_b"} - - -def test_union(): - fqn_a = SelectionCriteria.from_single_spec("fqn:model_a") - fqn_b = SelectionCriteria.from_single_spec("fqn:model_b") - fqn_c = SelectionCriteria.from_single_spec("fqn:model_c") - difference = SelectionUnion(components=[fqn_a, fqn_b, fqn_c]) - combined = difference.combine_selections( - [{"model_a", "model_b"}, {"model_b", "model_c"}, {"model_d"}] - ) - assert combined == {"model_a", "model_b", "model_c", "model_d"} diff --git a/tests/unit/test_infer_primary_key.py b/tests/unit/test_infer_primary_key.py deleted file mode 100644 index 8500c8ea7fe..00000000000 --- a/tests/unit/test_infer_primary_key.py +++ /dev/null @@ -1,195 +0,0 @@ -from dbt.artifacts.resources import TestMetadata -from dbt.contracts.graph.model_config import TestConfig -from dbt.contracts.graph.nodes import ColumnInfo -from dbt_common.contracts.constraints import ( - ColumnLevelConstraint, - ConstraintType, - ModelLevelConstraint, -) - -from .fixtures import generic_test_node, model_node - - -def test_no_primary_key(): - model = model_node() - assert model.infer_primary_key([]) == [] - - -def test_primary_key_model_constraint(): - model = model_node() - model.constraints = [ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk"])] - assertSameContents(model.infer_primary_key([]), ["pk"]) - - model.constraints = [ - ModelLevelConstraint(type=ConstraintType.primary_key, columns=["pk1", "pk2"]) - ] - assertSameContents(model.infer_primary_key([]), ["pk1", "pk2"]) - - -def test_primary_key_column_constraint(): - model = model_node() - model.columns = { - "column1": ColumnInfo( - "column1", constraints=[ColumnLevelConstraint(type=ConstraintType.primary_key)] - ), - "column2": ColumnInfo("column2"), - } - assertSameContents(model.infer_primary_key([]), ["column1"]) - - -def test_unique_non_null_single(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column1"}) - test3 = generic_test_node() - test3.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2"}) - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1"]) - - -def test_unique_non_null_multiple(): - model = model_node() - tests = [] - for i in range(2): - for enabled in [True, False]: - test1 = generic_test_node() - test1.test_metadata = TestMetadata( - name="unique", kwargs={"column_name": "column" + str(i) + str(enabled)} - ) - test1.config = TestConfig(enabled=enabled) - test2 = generic_test_node() - test2.test_metadata = TestMetadata( - name="not_null", kwargs={"column_name": "column" + str(i) + str(enabled)} - ) - test2.config = TestConfig(enabled=enabled) - tests.extend([test1, test2]) - - assertSameContents( - model.infer_primary_key(tests), - ["column0True", "column1True", "column0False", "column1False"], - ) - - -def test_enabled_unique_single(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column3"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1"]) - - -def test_enabled_unique_multiple(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) - - -def test_enabled_unique_combo_single(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata( - name="unique_combination_of_columns", - kwargs={"combination_of_columns": ["column1", "column2"]}, - ) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata( - name="unique_combination_of_columns", - kwargs={"combination_of_columns": ["column3", "column4"]}, - ) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) - - -def test_enabled_unique_combo_multiple(): - model = model_node() - test1 = generic_test_node() - test1.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} - ) - test2 = generic_test_node() - test2.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} - ) - - tests = [test1, test2] - assertSameContents( - model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] - ) - - -def test_disabled_unique_single(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.test_metadata = TestMetadata(name="not_null", kwargs={"column_name": "column2"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1"]) - - -def test_disabled_unique_multiple(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column1"}) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata(name="unique", kwargs={"column_name": "column2 || column3"}) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2 || column3"]) - - -def test_disabled_unique_combo_single(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} - ) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata( - name="random", kwargs={"combination_of_columns": ["column3", "column4"]} - ) - - tests = [test1, test2] - assertSameContents(model.infer_primary_key(tests), ["column1", "column2"]) - - -def test_disabled_unique_combo_multiple(): - model = model_node() - test1 = generic_test_node() - test1.config = TestConfig(enabled=False) - test1.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column1", "column2"]} - ) - test2 = generic_test_node() - test2.config = TestConfig(enabled=False) - test2.test_metadata = TestMetadata( - name="unique", kwargs={"combination_of_columns": ["column3", "column4"]} - ) - - tests = [test1, test2] - assertSameContents( - model.infer_primary_key(tests), ["column1", "column2", "column3", "column4"] - ) - - -def assertSameContents(list1, list2): - assert sorted(list1) == sorted(list2) diff --git a/tests/unit/test_inject_ctes.py b/tests/unit/test_inject_ctes.py deleted file mode 100644 index e6efdd4f1a6..00000000000 --- a/tests/unit/test_inject_ctes.py +++ /dev/null @@ -1,198 +0,0 @@ -import re - -from dbt.compilation import inject_ctes_into_sql -from dbt.contracts.graph.nodes import InjectedCTE - - -def norm_whitespace(string): - _RE_COMBINE_WHITESPACE = re.compile(r"\s+") - string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip() - return string - - -def test_inject_ctes_simple1(): - starting_sql = "select * from __dbt__cte__base" - ctes = [ - InjectedCTE( - id="model.test.base", - sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", - ) - ] - expected_sql = """with __dbt__cte__base as ( - select * from test16873767336887004702_test_ephemeral.seed - ) select * from __dbt__cte__base""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_simple2(): - starting_sql = "select * from __dbt__cte__ephemeral_level_two" - ctes = [ - InjectedCTE( - id="model.test.ephemeral_level_two", - sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873757769710148165_test_ephemeral"."source_table"\n)', - ) - ] - expected_sql = """with __dbt__cte__ephemeral_level_two as ( - select * from "dbt"."test16873757769710148165_test_ephemeral"."source_table" - ) select * from __dbt__cte__ephemeral_level_two""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_multiple_ctes(): - - starting_sql = "select * from __dbt__cte__ephemeral" - ctes = [ - InjectedCTE( - id="model.test.ephemeral_level_two", - sql=' __dbt__cte__ephemeral_level_two as (\n\nselect * from "dbt"."test16873735573223965828_test_ephemeral"."source_table"\n)', - ), - InjectedCTE( - id="model.test.ephemeral", - sql=" __dbt__cte__ephemeral as (\n\nselect * from __dbt__cte__ephemeral_level_two\n)", - ), - ] - expected_sql = """with __dbt__cte__ephemeral_level_two as ( - select * from "dbt"."test16873735573223965828_test_ephemeral"."source_table" - ), __dbt__cte__ephemeral as ( - select * from __dbt__cte__ephemeral_level_two - ) select * from __dbt__cte__ephemeral""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_multiple_ctes_more_complex(): - starting_sql = """select * from __dbt__cte__female_only - union all - select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" - ctes = [ - InjectedCTE( - id="model.test.base", - sql=" __dbt__cte__base as (\n\n\nselect * from test16873757723266827902_test_ephemeral.seed\n)", - ), - InjectedCTE( - id="model.test.base_copy", - sql=" __dbt__cte__base_copy as (\n\n\nselect * from __dbt__cte__base\n)", - ), - InjectedCTE( - id="model.test.female_only", - sql=" __dbt__cte__female_only as (\n\n\nselect * from __dbt__cte__base_copy where gender = 'Female'\n)", - ), - ] - expected_sql = """with __dbt__cte__base as ( - select * from test16873757723266827902_test_ephemeral.seed - ), __dbt__cte__base_copy as ( - select * from __dbt__cte__base - ), __dbt__cte__female_only as ( - select * from __dbt__cte__base_copy where gender = 'Female' - ) select * from __dbt__cte__female_only - union all - select * from "dbt"."test16873757723266827902_test_ephemeral"."double_dependent" where gender = 'Male'""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_starting_with1(): - starting_sql = """ - with internal_cte as (select * from sessions) - select * from internal_cte - """ - ctes = [ - InjectedCTE( - id="cte_id_1", - sql="__dbt__cte__ephemeral as (select * from table)", - ), - InjectedCTE( - id="cte_id_2", - sql="__dbt__cte__events as (select id, type from events)", - ), - ] - expected_sql = """with __dbt__cte__ephemeral as (select * from table), - __dbt__cte__events as (select id, type from events), - internal_cte as (select * from sessions) - select * from internal_cte""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_starting_with2(): - starting_sql = """with my_other_cool_cte as ( - select id, name from __dbt__cte__ephemeral - where id > 1000 - ) - select name, id from my_other_cool_cte""" - ctes = [ - InjectedCTE( - id="model.singular_tests_ephemeral.ephemeral", - sql=' __dbt__cte__ephemeral as (\n\n\nwith my_cool_cte as (\n select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base"\n)\nselect id, name from my_cool_cte where id is not null\n)', - ) - ] - expected_sql = """with __dbt__cte__ephemeral as ( - with my_cool_cte as ( - select name, id from "dbt"."test16873917221900185954_test_singular_tests_ephemeral"."base" - ) - select id, name from my_cool_cte where id is not null - ), my_other_cool_cte as ( - select id, name from __dbt__cte__ephemeral - where id > 1000 - ) - select name, id from my_other_cool_cte""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_comment_with(): - # Test injection with a comment containing "with" - starting_sql = """ - --- This is sql with a comment - select * from __dbt__cte__base - """ - ctes = [ - InjectedCTE( - id="model.test.base", - sql=" __dbt__cte__base as (\n\n\nselect * from test16873767336887004702_test_ephemeral.seed\n)", - ) - ] - expected_sql = """with __dbt__cte__base as ( - select * from test16873767336887004702_test_ephemeral.seed - ) --- This is sql with a comment - select * from __dbt__cte__base""" - - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) - - -def test_inject_ctes_with_recursive(): - # Test injection with "recursive" keyword - starting_sql = """ - with recursive t(n) as ( - select * from __dbt__cte__first_ephemeral_model - union all - select n+1 from t where n < 100 - ) - select sum(n) from t - """ - ctes = [ - InjectedCTE( - id="model.test.first_ephemeral_model", - sql=" __dbt__cte__first_ephemeral_model as (\n\nselect 1 as fun\n)", - ) - ] - expected_sql = """with recursive __dbt__cte__first_ephemeral_model as ( - select 1 as fun - ), t(n) as ( - select * from __dbt__cte__first_ephemeral_model - union all - select n+1 from t where n < 100 - ) - select sum(n) from t - """ - generated_sql = inject_ctes_into_sql(starting_sql, ctes) - assert norm_whitespace(generated_sql) == norm_whitespace(expected_sql) diff --git a/tests/unit/test_manifest_selectors.py b/tests/unit/test_manifest_selectors.py deleted file mode 100644 index 36ab31a5874..00000000000 --- a/tests/unit/test_manifest_selectors.py +++ /dev/null @@ -1,201 +0,0 @@ -import textwrap -import unittest -from collections import OrderedDict - -import yaml - -from dbt.config.selectors import SelectorDict -from dbt.exceptions import DbtSelectorsError - - -def get_selector_dict(txt: str) -> OrderedDict: - txt = textwrap.dedent(txt) - dct = OrderedDict(yaml.safe_load(txt)) - return dct - - -class SelectorUnitTest(unittest.TestCase): - def test_compare_cli_non_cli(self): - dct = get_selector_dict( - """\ - selectors: - - name: nightly_diet_snowplow - description: "This uses more CLI-style syntax" - definition: - union: - - intersection: - - '@source:snowplow' - - 'tag:nightly' - - 'models/export' - - exclude: - - intersection: - - 'package:snowplow' - - 'config.materialized:incremental' - - export_performance_timing - - name: nightly_diet_snowplow_full - description: "This is a fuller YAML specification" - definition: - union: - - intersection: - - method: source - value: snowplow - childrens_parents: true - - method: tag - value: nightly - - method: path - value: models/export - - exclude: - - intersection: - - method: package - value: snowplow - - method: config.materialized - value: incremental - - method: fqn - value: export_performance_timing - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - with_strings = sel_dict["nightly_diet_snowplow"]["definition"] - no_strings = sel_dict["nightly_diet_snowplow_full"]["definition"] - self.assertEqual(with_strings, no_strings) - - def test_single_string_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: nightly_selector - definition: - 'tag:nightly' - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "tag", "value": "nightly"} - definition = sel_dict["nightly_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_single_key_value_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: nightly_selector - definition: - tag: nightly - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "tag", "value": "nightly"} - definition = sel_dict["nightly_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_parent_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: kpi_nightly_selector - definition: - '+exposure:kpi_nightly' - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "exposure", "value": "kpi_nightly", "parents": True} - definition = sel_dict["kpi_nightly_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_plus_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: my_model_children_selector - definition: - 'my_model+2' - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - expected = {"method": "fqn", "value": "my_model", "children": True, "children_depth": "2"} - definition = sel_dict["my_model_children_selector"]["definition"] - self.assertEqual(expected, definition) - - def test_selector_definition(self): - dct = get_selector_dict( - """\ - selectors: - - name: default - definition: - union: - - intersection: - - tag: foo - - tag: bar - - name: inherited - definition: - method: selector - value: default - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list(dct["selectors"]) - assert sel_dict - definition = sel_dict["default"]["definition"] - expected = sel_dict["inherited"]["definition"] - self.assertEqual(expected, definition) - - def test_selector_definition_with_exclusion(self): - dct = get_selector_dict( - """\ - selectors: - - name: default - definition: - union: - - intersection: - - tag: foo - - tag: bar - - name: inherited - definition: - union: - - method: selector - value: default - - exclude: - - tag: bar - - name: comparison - definition: - union: - - union: - - intersection: - - tag: foo - - tag: bar - - exclude: - - tag: bar - """ - ) - - sel_dict = SelectorDict.parse_from_selectors_list((dct["selectors"])) - assert sel_dict - definition = sel_dict["inherited"]["definition"] - expected = sel_dict["comparison"]["definition"] - self.assertEqual(expected, definition) - - def test_missing_selector(self): - dct = get_selector_dict( - """\ - selectors: - - name: inherited - definition: - method: selector - value: default - """ - ) - with self.assertRaises(DbtSelectorsError) as err: - SelectorDict.parse_from_selectors_list((dct["selectors"])) - - self.assertEqual( - "Existing selector definition for default not found.", str(err.exception.msg) - ) diff --git a/tests/unit/test_semver.py b/tests/unit/test_semver.py deleted file mode 100644 index ae48e592fc0..00000000000 --- a/tests/unit/test_semver.py +++ /dev/null @@ -1,298 +0,0 @@ -import itertools -import unittest -from typing import List - -from dbt_common.exceptions import VersionsNotCompatibleError -from dbt_common.semver import ( - UnboundedVersionSpecifier, - VersionRange, - VersionSpecifier, - filter_installable, - reduce_versions, - resolve_to_specific_version, - versions_compatible, -) - - -def semver_regex_versioning(versions: List[str]) -> bool: - for version_string in versions: - try: - VersionSpecifier.from_version_string(version_string) - except Exception: - return False - return True - - -def create_range(start_version_string, end_version_string): - start = UnboundedVersionSpecifier() - end = UnboundedVersionSpecifier() - - if start_version_string is not None: - start = VersionSpecifier.from_version_string(start_version_string) - - if end_version_string is not None: - end = VersionSpecifier.from_version_string(end_version_string) - - return VersionRange(start=start, end=end) - - -class TestSemver(unittest.TestCase): - def assertVersionSetResult(self, inputs, output_range): - expected = create_range(*output_range) - - for permutation in itertools.permutations(inputs): - self.assertEqual(reduce_versions(*permutation), expected) - - def assertInvalidVersionSet(self, inputs): - for permutation in itertools.permutations(inputs): - with self.assertRaises(VersionsNotCompatibleError): - reduce_versions(*permutation) - - def test__versions_compatible(self): - self.assertTrue(versions_compatible("0.0.1", "0.0.1")) - self.assertFalse(versions_compatible("0.0.1", "0.0.2")) - self.assertTrue(versions_compatible(">0.0.1", "0.0.2")) - self.assertFalse(versions_compatible("0.4.5a1", "0.4.5a2")) - - def test__semver_regex_versions(self): - self.assertTrue( - semver_regex_versioning( - [ - "0.0.4", - "1.2.3", - "10.20.30", - "1.1.2-prerelease+meta", - "1.1.2+meta", - "1.1.2+meta-valid", - "1.0.0-alpha", - "1.0.0-beta", - "1.0.0-alpha.beta", - "1.0.0-alpha.beta.1", - "1.0.0-alpha.1", - "1.0.0-alpha0.valid", - "1.0.0-alpha.0valid", - "1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay", - "1.0.0-rc.1+build.1", - "2.0.0-rc.1+build.123", - "1.2.3-beta", - "10.2.3-DEV-SNAPSHOT", - "1.2.3-SNAPSHOT-123", - "1.0.0", - "2.0.0", - "1.1.7", - "2.0.0+build.1848", - "2.0.1-alpha.1227", - "1.0.0-alpha+beta", - "1.2.3----RC-SNAPSHOT.12.9.1--.12+788", - "1.2.3----R-S.12.9.1--.12+meta", - "1.2.3----RC-SNAPSHOT.12.9.1--.12", - "1.0.0+0.build.1-rc.10000aaa-kk-0.1", - "99999999999999999999999.999999999999999999.99999999999999999", - "1.0.0-0A.is.legal", - ] - ) - ) - - self.assertFalse( - semver_regex_versioning( - [ - "1", - "1.2", - "1.2.3-0123", - "1.2.3-0123.0123", - "1.1.2+.123", - "+invalid", - "-invalid", - "-invalid+invalid", - "-invalid.01", - "alpha", - "alpha.beta", - "alpha.beta.1", - "alpha.1", - "alpha+beta", - "alpha_beta", - "alpha.", - "alpha..", - "beta", - "1.0.0-alpha_beta", - "-alpha.", - "1.0.0-alpha..", - "1.0.0-alpha..1", - "1.0.0-alpha...1", - "1.0.0-alpha....1", - "1.0.0-alpha.....1", - "1.0.0-alpha......1", - "1.0.0-alpha.......1", - "01.1.1", - "1.01.1", - "1.1.01", - "1.2", - "1.2.3.DEV", - "1.2-SNAPSHOT", - "1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788", - "1.2-RC-SNAPSHOT", - "-1.0.3-gamma+b7718", - "+justmeta", - "9.8.7+meta+meta", - "9.8.7-whatever+meta+meta", - "99999999999999999999999.999999999999999999.99999999999999999----RC-SNAPSHOT.12.09.1--------------------------------..12", - ] - ) - ) - - def test__reduce_versions(self): - self.assertVersionSetResult(["0.0.1", "0.0.1"], ["=0.0.1", "=0.0.1"]) - - self.assertVersionSetResult(["0.0.1"], ["=0.0.1", "=0.0.1"]) - - self.assertVersionSetResult([">0.0.1"], [">0.0.1", None]) - - self.assertVersionSetResult(["<0.0.1"], [None, "<0.0.1"]) - - self.assertVersionSetResult([">0.0.1", "0.0.2"], ["=0.0.2", "=0.0.2"]) - - self.assertVersionSetResult(["0.0.2", ">=0.0.2"], ["=0.0.2", "=0.0.2"]) - - self.assertVersionSetResult([">0.0.1", ">0.0.2", ">0.0.3"], [">0.0.3", None]) - - self.assertVersionSetResult([">0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"]) - - self.assertVersionSetResult([">0.0.1", "0.0.2", "<0.0.3"], ["=0.0.2", "=0.0.2"]) - - self.assertVersionSetResult([">0.0.1", ">=0.0.1", "<0.0.3"], [">0.0.1", "<0.0.3"]) - - self.assertVersionSetResult([">0.0.1", "<0.0.3", "<=0.0.3"], [">0.0.1", "<0.0.3"]) - - self.assertVersionSetResult([">0.0.1", ">0.0.2", "<0.0.3", "<0.0.4"], [">0.0.2", "<0.0.3"]) - - self.assertVersionSetResult(["<=0.0.3", ">=0.0.3"], [">=0.0.3", "<=0.0.3"]) - - self.assertInvalidVersionSet([">0.0.2", "0.0.1"]) - self.assertInvalidVersionSet([">0.0.2", "0.0.2"]) - self.assertInvalidVersionSet(["<0.0.2", "0.0.2"]) - self.assertInvalidVersionSet(["<0.0.2", ">0.0.3"]) - self.assertInvalidVersionSet(["<=0.0.3", ">0.0.3"]) - self.assertInvalidVersionSet(["<0.0.3", ">=0.0.3"]) - self.assertInvalidVersionSet(["<0.0.3", ">0.0.3"]) - - def test__resolve_to_specific_version(self): - self.assertEqual( - resolve_to_specific_version(create_range(">0.0.1", None), ["0.0.1", "0.0.2"]), "0.0.2" - ) - - self.assertEqual( - resolve_to_specific_version(create_range(">=0.0.2", None), ["0.0.1", "0.0.2"]), "0.0.2" - ) - - self.assertEqual( - resolve_to_specific_version(create_range(">=0.0.3", None), ["0.0.1", "0.0.2"]), None - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=0.0.3", "<0.0.5"), ["0.0.3", "0.0.4", "0.0.5"] - ), - "0.0.4", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(None, "<=0.0.5"), ["0.0.3", "0.1.4", "0.0.5"] - ), - "0.0.5", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range("=0.4.5a2", "=0.4.5a2"), ["0.4.5a1", "0.4.5a2"] - ), - "0.4.5a2", - ) - - self.assertEqual( - resolve_to_specific_version(create_range("=0.7.6", "=0.7.6"), ["0.7.6-b1", "0.7.6"]), - "0.7.6", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"] - ), - "1.2.0a1", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", "<1.2.0"), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1"] - ), - "1.1.0", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", None), ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"] - ), - "1.2.0", - ) - - self.assertEqual( - resolve_to_specific_version( - create_range(">=1.0.0", "<1.2.0"), - ["1.0.0", "1.1.0a1", "1.1.0", "1.2.0a1", "1.2.0"], - ), - "1.1.0", - ) - - self.assertEqual( - resolve_to_specific_version( - # https://github.com/dbt-labs/dbt-core/issues/7039 - # 10 is greater than 9 - create_range(">0.9.0", "<0.10.0"), - ["0.9.0", "0.9.1", "0.10.0"], - ), - "0.9.1", - ) - - def test__filter_installable(self): - installable = filter_installable( - [ - "1.1.0", - "1.2.0a1", - "1.0.0", - "2.1.0-alpha", - "2.2.0asdf", - "2.1.0", - "2.2.0", - "2.2.0-fishtown-beta", - "2.2.0-2", - ], - install_prerelease=True, - ) - expected = [ - "1.0.0", - "1.1.0", - "1.2.0a1", - "2.1.0-alpha", - "2.1.0", - "2.2.0-2", - "2.2.0asdf", - "2.2.0-fishtown-beta", - "2.2.0", - ] - assert installable == expected - - installable = filter_installable( - [ - "1.1.0", - "1.2.0a1", - "1.0.0", - "2.1.0-alpha", - "2.2.0asdf", - "2.1.0", - "2.2.0", - "2.2.0-fishtown-beta", - ], - install_prerelease=False, - ) - expected = ["1.0.0", "1.1.0", "2.1.0", "2.2.0"] - assert installable == expected From b22200f007806e97f24651585c8dc1eb0f26f6f9 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Tue, 21 May 2024 10:02:52 -0700 Subject: [PATCH 07/10] add back test_events --- tests/unit/test_events.py | 574 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 574 insertions(+) create mode 100644 tests/unit/test_events.py diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py new file mode 100644 index 00000000000..bd9892bcc6c --- /dev/null +++ b/tests/unit/test_events.py @@ -0,0 +1,574 @@ +import logging +import re +from argparse import Namespace +from typing import TypeVar + +import pytest + +from dbt.adapters.events import types as adapter_types +from dbt.adapters.events.logging import AdapterLogger +from dbt.artifacts.schemas.results import RunStatus, TimingInfo +from dbt.artifacts.schemas.run import RunResult +from dbt.events import types as core_types +from dbt.events.base_types import ( + CoreBaseEvent, + DebugLevel, + DynamicLevel, + ErrorLevel, + InfoLevel, + TestLevel, + WarnLevel, +) +from dbt.events.types import RunResultError +from dbt.flags import set_from_args +from dbt.task.printer import print_run_result_error +from dbt_common.events import types +from dbt_common.events.base_types import msg_from_base_event +from dbt_common.events.event_manager import EventManager, TestEventManager +from dbt_common.events.event_manager_client import ctx_set_event_manager +from dbt_common.events.functions import msg_to_dict, msg_to_json +from dbt_common.events.helpers import get_json_string_utcnow + +set_from_args(Namespace(WARN_ERROR=False), None) + + +# takes in a class and finds any subclasses for it +def get_all_subclasses(cls): + all_subclasses = [] + for subclass in cls.__subclasses__(): + if subclass not in [TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel, DynamicLevel]: + all_subclasses.append(subclass) + all_subclasses.extend(get_all_subclasses(subclass)) + return set(all_subclasses) + + +class TestAdapterLogger: + # this interface is documented for adapter maintainers to plug into + # so we should test that it at the very least doesn't explode. + def test_basic_adapter_logging_interface(self): + logger = AdapterLogger("dbt_tests") + logger.debug("debug message") + logger.info("info message") + logger.warning("warning message") + logger.error("error message") + logger.exception("exception message") + logger.critical("exception message") + + # python loggers allow deferring string formatting via this signature: + def test_formatting(self): + logger = AdapterLogger("dbt_tests") + # tests that it doesn't throw + logger.debug("hello {}", "world") + + # enters lower in the call stack to test that it formats correctly + event = adapter_types.AdapterEventDebug( + name="dbt_tests", base_msg="hello {}", args=["world"] + ) + assert "hello world" in event.message() + + # tests that it doesn't throw + logger.debug("1 2 {}", "3") + + # enters lower in the call stack to test that it formats correctly + event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg="1 2 {}", args=[3]) + assert "1 2 3" in event.message() + + # tests that it doesn't throw + logger.debug("boop{x}boop") + + # enters lower in the call stack to test that it formats correctly + # in this case it's that we didn't attempt to replace anything since there + # were no args passed after the initial message + event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg="boop{x}boop", args=[]) + assert "boop{x}boop" in event.message() + + # ensure AdapterLogger and subclasses makes all base_msg members + # of type string; when someone writes logger.debug(a) where a is + # any non-string object + event = adapter_types.AdapterEventDebug(name="dbt_tests", base_msg=[1, 2, 3], args=[3]) + assert isinstance(event.base_msg, str) + + event = core_types.JinjaLogDebug(msg=[1, 2, 3]) + assert isinstance(event.msg, str) + + def test_set_adapter_dependency_log_level(self): + logger = AdapterLogger("dbt_tests") + package_log = logging.getLogger("test_package_log") + logger.set_adapter_dependency_log_level("test_package_log", "DEBUG") + package_log.debug("debug message") + + +class TestEventCodes: + + # checks to see if event codes are duplicated to keep codes singluar and clear. + # also checks that event codes follow correct namming convention ex. E001 + def test_event_codes(self): + all_concrete = get_all_subclasses(CoreBaseEvent) + all_codes = set() + + for event_cls in all_concrete: + code = event_cls.code(event_cls) + # must be in the form 1 capital letter, 3 digits + assert re.match("^[A-Z][0-9]{3}", code) + # cannot have been used already + assert ( + code not in all_codes + ), f"{code} is assigned more than once. Check types.py for duplicates." + all_codes.add(code) + + +sample_values = [ + # N.B. Events instantiated here include the module prefix in order to + # avoid having the entire list twice in the code. + # A - pre-project loading + core_types.MainReportVersion(version=""), + core_types.MainReportArgs(args={}), + core_types.MainTrackingUserState(user_state=""), + core_types.MissingProfileTarget(profile_name="", target_name=""), + core_types.InvalidOptionYAML(option_name="vars"), + core_types.LogDbtProjectError(), + core_types.LogDbtProfileError(), + core_types.StarterProjectPath(dir=""), + core_types.ConfigFolderDirectory(dir=""), + core_types.NoSampleProfileFound(adapter=""), + core_types.ProfileWrittenWithSample(name="", path=""), + core_types.ProfileWrittenWithTargetTemplateYAML(name="", path=""), + core_types.ProfileWrittenWithProjectTemplateYAML(name="", path=""), + core_types.SettingUpProfile(), + core_types.InvalidProfileTemplateYAML(), + core_types.ProjectNameAlreadyExists(name=""), + core_types.ProjectCreated(project_name=""), + # D - Deprecations ====================== + core_types.PackageRedirectDeprecation(old_name="", new_name=""), + core_types.PackageInstallPathDeprecation(), + core_types.ConfigSourcePathDeprecation(deprecated_path="", exp_path=""), + core_types.ConfigDataPathDeprecation(deprecated_path="", exp_path=""), + adapter_types.AdapterDeprecationWarning(old_name="", new_name=""), + core_types.MetricAttributesRenamed(metric_name=""), + core_types.ExposureNameDeprecation(exposure=""), + core_types.InternalDeprecation(name="", reason="", suggested_action="", version=""), + core_types.EnvironmentVariableRenamed(old_name="", new_name=""), + core_types.ConfigLogPathDeprecation(deprecated_path=""), + core_types.ConfigTargetPathDeprecation(deprecated_path=""), + adapter_types.CollectFreshnessReturnSignature(), + core_types.TestsConfigDeprecation(deprecated_path="", exp_path=""), + core_types.ProjectFlagsMovedDeprecation(), + core_types.SpacesInResourceNameDeprecation(unique_id="", level=""), + core_types.ResourceNamesWithSpacesDeprecation( + count_invalid_names=1, show_debug_hint=True, level="" + ), + core_types.PackageMaterializationOverrideDeprecation( + package_name="my_package", materialization_name="view" + ), + core_types.SourceFreshnessProjectHooksNotRun(), + # E - DB Adapter ====================== + adapter_types.AdapterEventDebug(), + adapter_types.AdapterEventInfo(), + adapter_types.AdapterEventWarning(), + adapter_types.AdapterEventError(), + adapter_types.AdapterRegistered(adapter_name="dbt-awesome", adapter_version="1.2.3"), + adapter_types.NewConnection(conn_type="", conn_name=""), + adapter_types.ConnectionReused(conn_name=""), + adapter_types.ConnectionLeftOpenInCleanup(conn_name=""), + adapter_types.ConnectionClosedInCleanup(conn_name=""), + adapter_types.RollbackFailed(conn_name=""), + adapter_types.ConnectionClosed(conn_name=""), + adapter_types.ConnectionLeftOpen(conn_name=""), + adapter_types.Rollback(conn_name=""), + adapter_types.CacheMiss(conn_name="", database="", schema=""), + adapter_types.ListRelations(database="", schema=""), + adapter_types.ConnectionUsed(conn_type="", conn_name=""), + adapter_types.SQLQuery(conn_name="", sql=""), + adapter_types.SQLQueryStatus(status="", elapsed=0.1), + adapter_types.SQLCommit(conn_name=""), + adapter_types.ColTypeChange( + orig_type="", + new_type="", + table={"database": "", "schema": "", "identifier": ""}, + ), + adapter_types.SchemaCreation(relation={"database": "", "schema": "", "identifier": ""}), + adapter_types.SchemaDrop(relation={"database": "", "schema": "", "identifier": ""}), + adapter_types.CacheAction( + action="adding_relation", + ref_key={"database": "", "schema": "", "identifier": ""}, + ref_key_2={"database": "", "schema": "", "identifier": ""}, + ), + adapter_types.CacheDumpGraph(before_after="before", action="rename", dump=dict()), + adapter_types.AdapterImportError(exc=""), + adapter_types.PluginLoadError(exc_info=""), + adapter_types.NewConnectionOpening(connection_state=""), + adapter_types.CodeExecution(conn_name="", code_content=""), + adapter_types.CodeExecutionStatus(status="", elapsed=0.1), + adapter_types.CatalogGenerationError(exc=""), + adapter_types.WriteCatalogFailure(num_exceptions=0), + adapter_types.CatalogWritten(path=""), + adapter_types.CannotGenerateDocs(), + adapter_types.BuildingCatalog(), + adapter_types.DatabaseErrorRunningHook(hook_type=""), + adapter_types.HooksRunning(num_hooks=0, hook_type=""), + adapter_types.FinishedRunningStats(stat_line="", execution="", execution_time=0), + adapter_types.ConstraintNotEnforced(constraint="", adapter=""), + adapter_types.ConstraintNotSupported(constraint="", adapter=""), + # I - Project parsing ====================== + core_types.InputFileDiffError(category="testing", file_id="my_file"), + core_types.InvalidValueForField(field_name="test", field_value="test"), + core_types.ValidationWarning(resource_type="model", field_name="access", node_name="my_macro"), + core_types.ParsePerfInfoPath(path=""), + core_types.PartialParsingErrorProcessingFile(file=""), + core_types.PartialParsingFile(file_id=""), + core_types.PartialParsingError(exc_info={}), + core_types.PartialParsingSkipParsing(), + core_types.UnableToPartialParse(reason="something went wrong"), + core_types.StateCheckVarsHash(vars="testing", target="testing", profile="testing"), + core_types.PartialParsingNotEnabled(), + core_types.ParsedFileLoadFailed(path="", exc="", exc_info=""), + core_types.PartialParsingEnabled(deleted=0, added=0, changed=0), + core_types.PartialParsingFile(file_id=""), + core_types.InvalidDisabledTargetInTestNode( + resource_type_title="", + unique_id="", + original_file_path="", + target_kind="", + target_name="", + target_package="", + ), + core_types.UnusedResourceConfigPath(unused_config_paths=[]), + core_types.SeedIncreased(package_name="", name=""), + core_types.SeedExceedsLimitSamePath(package_name="", name=""), + core_types.SeedExceedsLimitAndPathChanged(package_name="", name=""), + core_types.SeedExceedsLimitChecksumChanged(package_name="", name="", checksum_name=""), + core_types.UnusedTables(unused_tables=[]), + core_types.WrongResourceSchemaFile( + patch_name="", resource_type="", file_path="", plural_resource_type="" + ), + core_types.NoNodeForYamlKey(patch_name="", yaml_key="", file_path=""), + core_types.MacroNotFoundForPatch(patch_name=""), + core_types.NodeNotFoundOrDisabled( + original_file_path="", + unique_id="", + resource_type_title="", + target_name="", + target_kind="", + target_package="", + disabled="", + ), + core_types.JinjaLogWarning(), + core_types.JinjaLogInfo(msg=""), + core_types.JinjaLogDebug(msg=""), + core_types.UnpinnedRefNewVersionAvailable( + ref_node_name="", ref_node_package="", ref_node_version="", ref_max_version="" + ), + core_types.DeprecatedModel(model_name="", model_version="", deprecation_date=""), + core_types.DeprecatedReference( + model_name="", + ref_model_name="", + ref_model_package="", + ref_model_deprecation_date="", + ref_model_latest_version="", + ), + core_types.UpcomingReferenceDeprecation( + model_name="", + ref_model_name="", + ref_model_package="", + ref_model_deprecation_date="", + ref_model_latest_version="", + ), + core_types.UnsupportedConstraintMaterialization(materialized=""), + core_types.ParseInlineNodeError(exc=""), + core_types.SemanticValidationFailure(msg=""), + core_types.UnversionedBreakingChange( + breaking_changes=[], + model_name="", + model_file_path="", + contract_enforced_disabled=True, + columns_removed=[], + column_type_changes=[], + enforced_column_constraint_removed=[], + enforced_model_constraint_removed=[], + materialization_changed=[], + ), + core_types.WarnStateTargetEqual(state_path=""), + core_types.FreshnessConfigProblem(msg=""), + core_types.SemanticValidationFailure(msg=""), + # M - Deps generation ====================== + core_types.GitSparseCheckoutSubdirectory(subdir=""), + core_types.GitProgressCheckoutRevision(revision=""), + core_types.GitProgressUpdatingExistingDependency(dir=""), + core_types.GitProgressPullingNewDependency(dir=""), + core_types.GitNothingToDo(sha=""), + core_types.GitProgressUpdatedCheckoutRange(start_sha="", end_sha=""), + core_types.GitProgressCheckedOutAt(end_sha=""), + core_types.RegistryProgressGETRequest(url=""), + core_types.RegistryProgressGETResponse(url="", resp_code=1234), + core_types.SelectorReportInvalidSelector(valid_selectors="", spec_method="", raw_spec=""), + core_types.DepsNoPackagesFound(), + core_types.DepsStartPackageInstall(package_name=""), + core_types.DepsInstallInfo(version_name=""), + core_types.DepsUpdateAvailable(version_latest=""), + core_types.DepsUpToDate(), + core_types.DepsListSubdirectory(subdirectory=""), + core_types.DepsNotifyUpdatesAvailable(packages=["my_pkg", "other_pkg"]), + types.RetryExternalCall(attempt=0, max=0), + types.RecordRetryException(exc=""), + core_types.RegistryIndexProgressGETRequest(url=""), + core_types.RegistryIndexProgressGETResponse(url="", resp_code=1234), + core_types.RegistryResponseUnexpectedType(response=""), + core_types.RegistryResponseMissingTopKeys(response=""), + core_types.RegistryResponseMissingNestedKeys(response=""), + core_types.RegistryResponseExtraNestedKeys(response=""), + core_types.DepsSetDownloadDirectory(path=""), + core_types.DepsLockUpdating(lock_filepath=""), + core_types.DepsAddPackage(package_name="", version="", packages_filepath=""), + core_types.DepsFoundDuplicatePackage(removed_package={}), + core_types.DepsScrubbedPackageName(package_name=""), + core_types.DepsUnpinned(revision="", git=""), + core_types.NoNodesForSelectionCriteria(spec_raw=""), + # Q - Node execution ====================== + core_types.RunningOperationCaughtError(exc=""), + core_types.CompileComplete(), + core_types.FreshnessCheckComplete(), + core_types.SeedHeader(header=""), + core_types.SQLRunnerException(exc=""), + core_types.LogTestResult( + name="", + index=0, + num_models=0, + execution_time=0, + num_failures=0, + ), + core_types.LogStartLine(description="", index=0, total=0), + core_types.LogModelResult( + description="", + status="", + index=0, + total=0, + execution_time=0, + ), + core_types.LogSnapshotResult( + status="", + description="", + cfg={}, + index=0, + total=0, + execution_time=0, + ), + core_types.LogSeedResult( + status="", + index=0, + total=0, + execution_time=0, + schema="", + relation="", + ), + core_types.LogFreshnessResult( + source_name="", + table_name="", + index=0, + total=0, + execution_time=0, + ), + core_types.LogNodeNoOpResult( + description="", + status="", + index=0, + total=0, + execution_time=0, + ), + core_types.LogCancelLine(conn_name=""), + core_types.DefaultSelector(name=""), + core_types.NodeStart(), + core_types.NodeFinished(), + core_types.QueryCancelationUnsupported(type=""), + core_types.ConcurrencyLine(num_threads=0, target_name=""), + core_types.WritingInjectedSQLForNode(), + core_types.NodeCompiling(), + core_types.NodeExecuting(), + core_types.LogHookStartLine( + statement="", + index=0, + total=0, + ), + core_types.LogHookEndLine( + statement="", + status="", + index=0, + total=0, + execution_time=0, + ), + core_types.SkippingDetails( + resource_type="", + schema="", + node_name="", + index=0, + total=0, + ), + core_types.NothingToDo(), + core_types.RunningOperationUncaughtError(exc=""), + core_types.EndRunResult(), + core_types.NoNodesSelected(), + core_types.CommandCompleted( + command="", + success=True, + elapsed=0.1, + completed_at=get_json_string_utcnow(), + ), + core_types.ShowNode(node_name="", preview="", is_inline=True, unique_id="model.test.my_model"), + core_types.CompiledNode( + node_name="", compiled="", is_inline=True, unique_id="model.test.my_model" + ), + # W - Node testing ====================== + core_types.CatchableExceptionOnRun(exc=""), + core_types.InternalErrorOnRun(build_path="", exc=""), + core_types.GenericExceptionOnRun(build_path="", unique_id="", exc=""), + core_types.NodeConnectionReleaseError(node_name="", exc=""), + core_types.FoundStats(stat_line=""), + # Z - misc ====================== + core_types.MainKeyboardInterrupt(), + core_types.MainEncounteredError(exc=""), + core_types.MainStackTrace(stack_trace=""), + types.SystemCouldNotWrite(path="", reason="", exc=""), + types.SystemExecutingCmd(cmd=[""]), + types.SystemStdOut(bmsg=str(b"")), + types.SystemStdErr(bmsg=str(b"")), + types.SystemReportReturnCode(returncode=0), + core_types.TimingInfoCollected(), + core_types.LogDebugStackTrace(), + core_types.CheckCleanPath(path=""), + core_types.ConfirmCleanPath(path=""), + core_types.ProtectedCleanPath(path=""), + core_types.FinishedCleanPaths(), + core_types.OpenCommand(open_cmd="", profiles_dir=""), + core_types.RunResultWarning(resource_type="", node_name="", path=""), + core_types.RunResultFailure(resource_type="", node_name="", path=""), + core_types.StatsLine(stats={"error": 0, "skip": 0, "pass": 0, "warn": 0, "total": 0}), + core_types.RunResultError(msg=""), + core_types.RunResultErrorNoMessage(status=""), + core_types.SQLCompiledPath(path=""), + core_types.CheckNodeTestFailure(relation_name=""), + core_types.EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False), + core_types.LogSkipBecauseError(schema="", relation="", index=0, total=0), + core_types.EnsureGitInstalled(), + core_types.DepsCreatingLocalSymlink(), + core_types.DepsSymlinkNotAvailable(), + core_types.DisableTracking(), + core_types.SendingEvent(kwargs=""), + core_types.SendEventFailure(), + core_types.FlushEvents(), + core_types.FlushEventsFailure(), + types.Formatting(), + core_types.TrackingInitializeFailure(), + core_types.RunResultWarningMessage(), + core_types.DebugCmdOut(), + core_types.DebugCmdResult(), + core_types.ListCmdOut(), + types.Note(msg="This is a note."), + core_types.ResourceReport(), +] + + +class TestEventJSONSerialization: + + # attempts to test that every event is serializable to json. + # event types that take `Any` are not possible to test in this way since some will serialize + # just fine and others won't. + def test_all_serializable(self): + all_non_abstract_events = set( + get_all_subclasses(CoreBaseEvent), + ) + all_event_values_list = list(map(lambda x: x.__class__, sample_values)) + diff = all_non_abstract_events.difference(set(all_event_values_list)) + assert ( + not diff + ), f"{diff}test is missing concrete values in `sample_values`. Please add the values for the aforementioned event classes" + + # make sure everything in the list is a value not a type + for event in sample_values: + assert type(event) != type + + # if we have everything we need to test, try to serialize everything + count = 0 + for event in sample_values: + msg = msg_from_base_event(event) + print(f"--- msg: {msg.info.name}") + # Serialize to dictionary + try: + msg_to_dict(msg) + except Exception as e: + raise Exception( + f"{event} can not be converted to a dict. Originating exception: {e}" + ) + # Serialize to json + try: + msg_to_json(msg) + except Exception as e: + raise Exception(f"{event} is not serializable to json. Originating exception: {e}") + # Serialize to binary + try: + msg.SerializeToString() + except Exception as e: + raise Exception( + f"{event} is not serializable to binary protobuf. Originating exception: {e}" + ) + count += 1 + print(f"--- Found {count} events") + + +T = TypeVar("T") + + +def test_date_serialization(): + ti = TimingInfo("test") + ti.begin() + ti.end() + ti_dict = ti.to_dict() + assert ti_dict["started_at"].endswith("Z") + assert ti_dict["completed_at"].endswith("Z") + + +def test_bad_serialization(): + """Tests that bad serialization enters the proper exception handling + + When pytest is in use the exception handling of `BaseEvent` raises an + exception. When pytest isn't present, it fires a Note event. Thus to test + that bad serializations are properly handled, the best we can do is test + that the exception handling path is used. + """ + + with pytest.raises(Exception) as excinfo: + types.Note(param_event_doesnt_have="This should break") + + assert ( + str(excinfo.value) + == "[Note]: Unable to parse dict {'param_event_doesnt_have': 'This should break'}" + ) + + +def test_single_run_error(): + + try: + # Add a recording event manager to the context, so we can test events. + event_mgr = TestEventManager() + ctx_set_event_manager(event_mgr) + + error_result = RunResult( + status=RunStatus.Error, + timing=[], + thread_id="", + execution_time=0.0, + node=None, + adapter_response=dict(), + message="oh no!", + failures=[], + ) + + print_run_result_error(error_result) + events = [e for e in event_mgr.event_history if isinstance(e[0], RunResultError)] + + assert len(events) == 1 + assert events[0][0].msg == "oh no!" + + finally: + # Set an empty event manager unconditionally on exit. This is an early + # attempt at unit testing events, and we need to think about how it + # could be done in a thread safe way in the long run. + ctx_set_event_manager(EventManager()) From 7d5a53e5f41551a28e4fd256b4b919e156b67336 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Tue, 21 May 2024 10:05:43 -0700 Subject: [PATCH 08/10] nits --- tests/unit/clients/__init__.py | 0 tests/unit/deps/__init__.py | 0 tests/unit/events/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit/clients/__init__.py create mode 100644 tests/unit/deps/__init__.py create mode 100644 tests/unit/events/__init__.py diff --git a/tests/unit/clients/__init__.py b/tests/unit/clients/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/deps/__init__.py b/tests/unit/deps/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/events/__init__.py b/tests/unit/events/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 85f3bec8452a9edf4a6c836552c98cd95d767e84 Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Tue, 21 May 2024 13:44:26 -0700 Subject: [PATCH 09/10] update import --- tests/unit/parser/test_unit_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/parser/test_unit_tests.py b/tests/unit/parser/test_unit_tests.py index 15f018f6446..e8725bed718 100644 --- a/tests/unit/parser/test_unit_tests.py +++ b/tests/unit/parser/test_unit_tests.py @@ -5,10 +5,9 @@ from dbt.contracts.graph.unparsed import UnitTestOutputFixture from dbt.parser import SchemaParser from dbt.parser.unit_tests import UnitTestParser +from tests.unit.parser.test_parser import SchemaParserTest, assertEqualNodes from tests.unit.utils import MockNode -from .test_parser import SchemaParserTest, assertEqualNodes - UNIT_TEST_MODEL_NOT_FOUND_SOURCE = """ unit_tests: - name: test_my_model_doesnt_exist From 9bceb37f3723d3a0162ee721035ea535efa7c2ff Mon Sep 17 00:00:00 2001 From: Chenyu Li Date: Tue, 21 May 2024 13:52:18 -0700 Subject: [PATCH 10/10] add back test --- tests/unit/contracts/graph/test_unparsed.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/contracts/graph/test_unparsed.py b/tests/unit/contracts/graph/test_unparsed.py index 0368e9c2770..90fa2fcf8a7 100644 --- a/tests/unit/contracts/graph/test_unparsed.py +++ b/tests/unit/contracts/graph/test_unparsed.py @@ -15,6 +15,7 @@ from dbt.artifacts.schemas.results import FreshnessStatus from dbt.contracts.graph.unparsed import ( Docs, + HasColumnTests, UnparsedColumn, UnparsedDocumentationFile, UnparsedExposure, @@ -30,7 +31,9 @@ UnparsedSourceTableDefinition, UnparsedVersion, ) +from dbt.exceptions import ParsingError from dbt.node_types import NodeType +from dbt.parser.schemas import ParserRef from tests.unit.utils import ContractTestCase @@ -983,3 +986,12 @@ def test_bad_version_no_v(self): ) def test_unparsed_version_lt(left, right, expected_lt): assert (UnparsedVersion(left) < UnparsedVersion(right)) == expected_lt + + +def test_column_parse(): + unparsed_col = HasColumnTests( + columns=[UnparsedColumn(name="TestCol", constraints=[{"type": "!INVALID!"}])] + ) + + with pytest.raises(ParsingError): + ParserRef.from_target(unparsed_col)