From bd6289443006b6cdd7386ebe1823492f7d53966f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 10:39:05 +0200 Subject: [PATCH 01/10] Add project name to default search packages We prefer macros in the project over the ones in the namespace (package) --- core/dbt/context/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index e1115cae9bd..5211f0a17c9 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -144,7 +144,7 @@ def dispatch( elif isinstance(namespace, str): search_packages = self._adapter.config.get_macro_search_order(namespace) if not search_packages and namespace in self._adapter.config.dependencies: - search_packages = [namespace] + search_packages = [self.config.project, namespace] else: # Not a string and not None so must be a list raise CompilationException( From 8195adba310ab5cb94d55af48dbc3aa6c6b05910 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 10:42:24 +0200 Subject: [PATCH 02/10] Add change to change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e28868e1930..9c00f4cdc36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Make finding disabled nodes more consistent ([#4069](https://github.com/dbt-labs/dbt-core/issues/4069), [#4073](https://github.com/dbt-labas/dbt-core/pull/4073)) - Remove connection from `render_with_context` during parsing, thereby removing misleading log message ([#3137](https://github.com/dbt-labs/dbt-core/issues/3137), [#4062](https://github.com/dbt-labas/dbt-core/pull/4062)) - Wait for postgres docker container to be ready in `setup_db.sh`. ([#3876](https://github.com/dbt-labs/dbt-core/issues/3876), [#3908](https://github.com/dbt-labs/dbt-core/pull/3908)) +- Prefer local macros over the ones in a package ([#4106](https://github.com/dbt-labs/dbt-core/issues/4106), [#4114](https://github.com/dbt-labs/dbt-core/pull/4114)) Contributors: - [@sungchun12](https://github.com/sungchun12) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) From 6eb433ffa0473c7c8ac5afa9e2c0166b77ae671d Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 10:51:01 +0200 Subject: [PATCH 03/10] Use project_name instead of project --- core/dbt/context/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 5211f0a17c9..793d68f0900 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -144,7 +144,7 @@ def dispatch( elif isinstance(namespace, str): search_packages = self._adapter.config.get_macro_search_order(namespace) if not search_packages and namespace in self._adapter.config.dependencies: - search_packages = [self.config.project, namespace] + search_packages = [self.config.project_name, namespace] else: # Not a string and not None so must be a list raise CompilationException( From 25447419d8387fa601f4bafaa10a27bde93a742f Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 11:55:15 +0200 Subject: [PATCH 04/10] Raise compilation error if no macros are found --- core/dbt/context/providers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 793d68f0900..74334dad36f 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -162,10 +162,10 @@ def dispatch( macro = self._namespace.get_from_package( package_name, search_name ) - except CompilationException as exc: - raise CompilationException( - f'In dispatch: {exc.msg}', - ) from exc + except CompilationException: + # Only raise CompilationException if macro is not found in + # any package + macro = None if package_name is None: attempts.append(search_name) From e85c2c747e707dff923af0e6d3a1b2eed8041734 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 12:04:06 +0200 Subject: [PATCH 05/10] Update change log line --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c00f4cdc36..f6a23821a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ - Make finding disabled nodes more consistent ([#4069](https://github.com/dbt-labs/dbt-core/issues/4069), [#4073](https://github.com/dbt-labas/dbt-core/pull/4073)) - Remove connection from `render_with_context` during parsing, thereby removing misleading log message ([#3137](https://github.com/dbt-labs/dbt-core/issues/3137), [#4062](https://github.com/dbt-labas/dbt-core/pull/4062)) - Wait for postgres docker container to be ready in `setup_db.sh`. ([#3876](https://github.com/dbt-labs/dbt-core/issues/3876), [#3908](https://github.com/dbt-labs/dbt-core/pull/3908)) -- Prefer local macros over the ones in a package ([#4106](https://github.com/dbt-labs/dbt-core/issues/4106), [#4114](https://github.com/dbt-labs/dbt-core/pull/4114)) +- Prefer macros defined in the project over the ones in a package by default ([#4106](https://github.com/dbt-labs/dbt-core/issues/4106), [#4114](https://github.com/dbt-labs/dbt-core/pull/4114)) Contributors: - [@sungchun12](https://github.com/sungchun12) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) From ed7acbbdc159b45693733214a6a22a0f532907fe Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 14:11:42 +0200 Subject: [PATCH 06/10] Add test for package macro override --- .../macros.sql | 3 +++ .../016_macro_tests/test_macros.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql diff --git a/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql b/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql new file mode 100644 index 00000000000..bb26b6124e6 --- /dev/null +++ b/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql @@ -0,0 +1,3 @@ +{% macro postgress__get_columns_in_relation(relation) %} + {{ return('a string') }} +{% endmacro %} diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index 2dfe2a3f303..426f271d0b1 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -118,6 +118,29 @@ def test_postgres_overrides(self): self.run_dbt() +class TestMacroOverridePackage(DBTIntegrationTest): + @property + def schema(self): + return "test_macros_016" + + @property + def models(self): + return 'override-get-columns-models' + + @property + def project_config(self): + return { + 'config-version': 2, + 'macro-paths': ['override-postgres-get-columns-macros'], + } + + @use_profile('postgres') + def test_postgres_overrides(self): + # the first time, the model doesn't exist + self.run_dbt() + self.run_dbt() + + class TestDispatchMacroOverrideBuiltin(TestMacroOverrideBuiltin): # test the same functionality as above, but this time, # dbt.get_columns_in_relation will dispatch to a default__ macro From fe50126e4ca8e08c20a74b61cd381d6055d47d1b Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 14:25:37 +0200 Subject: [PATCH 07/10] Add JCZuurmond to contributor --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a23821a3c..3621aeadaa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Contributors: - [@matt-winkler](https://github.com/matt-winkler) ([#4017](https://github.com/dbt-labs/dbt/pull/4017)) - [@NiallRees](https://github.com/NiallRees) ([#3625](https://github.com/dbt-labs/dbt/pull/3625)) - [@rvacaru](https://github.com/rvacaru) ([#3908](https://github.com/dbt-labs/dbt/pull/3908)) +- [@JCZuurmond](https://github.com/jczuurmond) ([#4114](https://github.com/dbt-labs/dbt-core/pull/4114)) ## dbt-core 1.0.0b1 (October 11, 2021) From 396868b4377c654873981abca51d3412ea925344 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 14:46:20 +0200 Subject: [PATCH 08/10] Fix typos --- .../override-postgres-get-columns-macros/macros.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql b/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql index bb26b6124e6..4de53ba4a18 100644 --- a/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql +++ b/test/integration/016_macro_tests/override-postgres-get-columns-macros/macros.sql @@ -1,3 +1,3 @@ -{% macro postgress__get_columns_in_relation(relation) %} +{% macro postgres__get_columns_in_relation(relation) %} {{ return('a string') }} {% endmacro %} From 8bc28702ad3c9483155407d4105277ac32c587c4 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 14:55:07 +0200 Subject: [PATCH 09/10] Add test that should not over ride the package --- .../016_macro_tests/test_macros.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index 426f271d0b1..61105a80e03 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -141,6 +141,30 @@ def test_postgres_overrides(self): self.run_dbt() +class TestMacroNotOverridePackage(DBTIntegrationTest): + @property + def schema(self): + return "test_macros_016" + + @property + def models(self): + return 'override-get-columns-models' + + @property + def project_config(self): + return { + 'config-version': 2, + 'macro-paths': ['override-postgres-get-columns-macros'], + 'dispatch': [{'macro_namespace': 'dbt', 'search_order': ['dbt']}], + } + + @use_profile('postgres') + def test_postgres_overrides(self): + # the first time, the model doesn't exist + self.run_dbt(expect_pass=False) + self.run_dbt(expect_pass=False) + + class TestDispatchMacroOverrideBuiltin(TestMacroOverrideBuiltin): # test the same functionality as above, but this time, # dbt.get_columns_in_relation will dispatch to a default__ macro From 4f5690b90483ce7a3e06b4538d77f3d77f9d1a73 Mon Sep 17 00:00:00 2001 From: Cor Zuurmond Date: Fri, 22 Oct 2021 15:03:13 +0200 Subject: [PATCH 10/10] Add doc string to tests --- test/integration/016_macro_tests/test_macros.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/integration/016_macro_tests/test_macros.py b/test/integration/016_macro_tests/test_macros.py index 61105a80e03..6314da09f81 100644 --- a/test/integration/016_macro_tests/test_macros.py +++ b/test/integration/016_macro_tests/test_macros.py @@ -119,6 +119,11 @@ def test_postgres_overrides(self): class TestMacroOverridePackage(DBTIntegrationTest): + """ + The macro in `override-postgres-get-columns-macros` should override the + `get_columns_in_relation` macro by default. + """ + @property def schema(self): return "test_macros_016" @@ -142,6 +147,12 @@ def test_postgres_overrides(self): class TestMacroNotOverridePackage(DBTIntegrationTest): + """ + The macro in `override-postgres-get-columns-macros` does NOT override the + `get_columns_in_relation` macro because we tell dispatch to not look at the + postgres macros. + """ + @property def schema(self): return "test_macros_016"