From 8c99a48fa58fac091a639c2dd3771e1bba6e9446 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 7 Aug 2023 14:21:37 -0400 Subject: [PATCH 01/15] first pass: --sample --- core/dbt/adapters/base/relation.py | 8 +++++++- core/dbt/cli/main.py | 2 ++ core/dbt/cli/params.py | 8 ++++++++ core/dbt/context/providers.py | 10 +++++++--- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 67a50d9061f..d5132c6b8a1 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -1,6 +1,7 @@ from collections.abc import Hashable from dataclasses import dataclass, field from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set, Union, FrozenSet +import uuid from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode from dbt.contracts.relation import ( @@ -36,6 +37,7 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False + sample: Optional[int] = None # register relation types that can be renamed for the purpose of replacing relations using stages and backups # adding a relation type here also requires defining the associated rename macro @@ -192,7 +194,11 @@ def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[s def render(self) -> str: # if there is nothing set, this will return the empty string. - return ".".join(part for _, part in self._render_iterator() if part is not None) + rendered_parts = ".".join(part for _, part in self._render_iterator() if part is not None) + if self.sample and rendered_parts: + alias = f"_dbt_sample_{uuid.uuid4().hex.upper()[:6]}" + return f"(select * from {rendered_parts} limit {self.sample}) {alias}" + return rendered_parts def quoted(self, identifier): return "{quote_char}{identifier}{quote_char}".format( diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 7d4560a7910..97a48ce4426 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -342,6 +342,7 @@ def docs_serve(ctx, **kwargs): @p.profile @p.profiles_dir @p.project_dir +@p.sample @p.select @p.selector @p.inline @@ -599,6 +600,7 @@ def parse(ctx, **kwargs): @p.profile @p.profiles_dir @p.project_dir +@p.sample @p.select @p.selector @p.state diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 1898815a724..445e7995151 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -415,6 +415,14 @@ hidden=True, ) +sample = click.option( + "--sample", + envvar="DBT_SAMPLE", + help="Limit by sample rows when resolving dbt ref and sources.", + type=click.INT, + default=None, +) + model_decls = ("-m", "--models", "--model") select_decls = ("-s", "--select") select_attrs = { diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index febc21a546f..b7334118c39 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -531,9 +531,13 @@ def resolve( def create_relation(self, target_model: ManifestNode) -> RelationProxy: if target_model.is_ephemeral_model: self.model.set_cte(target_model.unique_id, None) - return self.Relation.create_ephemeral_from_node(self.config, target_model) + return self.Relation.create_ephemeral_from_node( + self.config, target_model, sample=self.config.args.sample + ) else: - return self.Relation.create_from(self.config, target_model) + return self.Relation.create_from( + self.config, target_model, sample=self.config.args.sample + ) def validate( self, @@ -590,7 +594,7 @@ def resolve(self, source_name: str, table_name: str): target_kind="source", disabled=(isinstance(target_source, Disabled)), ) - return self.Relation.create_from_source(target_source) + return self.Relation.create_from_source(target_source, sample=self.config.args.sample) # metric` implementations From 8180c08e971f3cfcddfd8006f87a0653e65ac491 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 1 Nov 2023 15:44:51 -0400 Subject: [PATCH 02/15] rename to --empty --- core/dbt/adapters/base/relation.py | 11 ++++++----- core/dbt/cli/main.py | 4 ++-- core/dbt/cli/params.py | 14 ++++++-------- core/dbt/context/providers.py | 6 +++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index d5132c6b8a1..b0025e18769 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -1,7 +1,6 @@ from collections.abc import Hashable from dataclasses import dataclass, field from typing import Optional, TypeVar, Any, Type, Dict, Iterator, Tuple, Set, Union, FrozenSet -import uuid from dbt.contracts.graph.nodes import SourceDefinition, ManifestNode, ResultNode, ParsedNode from dbt.contracts.relation import ( @@ -37,7 +36,7 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False - sample: Optional[int] = None + empty: bool = False # register relation types that can be renamed for the purpose of replacing relations using stages and backups # adding a relation type here also requires defining the associated rename macro @@ -51,6 +50,9 @@ class BaseRelation(FakeAPIObject, Hashable): # include/postgres/macros/relations/view/replace.sql::postgres__get_replace_view_sql() replaceable_relations: SerializableIterable = () + # empty subquery template used when rendering a relation in the context of an --empty invocation + empty_subquery_template = "(select * from {} where false limit 0) _dbt_empty_subq" + def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: return self.path.get_lowered_part(field) == value.lower() @@ -195,9 +197,8 @@ def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[s def render(self) -> str: # if there is nothing set, this will return the empty string. rendered_parts = ".".join(part for _, part in self._render_iterator() if part is not None) - if self.sample and rendered_parts: - alias = f"_dbt_sample_{uuid.uuid4().hex.upper()[:6]}" - return f"(select * from {rendered_parts} limit {self.sample}) {alias}" + if self.empty and rendered_parts: + return self.empty_subquery_template.format(rendered_parts) return rendered_parts def quoted(self, identifier): diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 97a48ce4426..2454a15a564 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -342,7 +342,7 @@ def docs_serve(ctx, **kwargs): @p.profile @p.profiles_dir @p.project_dir -@p.sample +@p.empty @p.select @p.selector @p.inline @@ -600,7 +600,7 @@ def parse(ctx, **kwargs): @p.profile @p.profiles_dir @p.project_dir -@p.sample +@p.empty @p.select @p.selector @p.state diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 445e7995151..3cea132d3a5 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -90,6 +90,12 @@ is_flag=True, ) +empty = click.option( + "--empty", + envvar="DBT_EMPTY", + help="If specified, limit input refs and sources to zero rows.", + is_flag=True, +) enable_legacy_logger = click.option( "--enable-legacy-logger/--no-enable-legacy-logger", @@ -415,14 +421,6 @@ hidden=True, ) -sample = click.option( - "--sample", - envvar="DBT_SAMPLE", - help="Limit by sample rows when resolving dbt ref and sources.", - type=click.INT, - default=None, -) - model_decls = ("-m", "--models", "--model") select_decls = ("-s", "--select") select_attrs = { diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index b7334118c39..dcaed7256e2 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -532,11 +532,11 @@ def create_relation(self, target_model: ManifestNode) -> RelationProxy: if target_model.is_ephemeral_model: self.model.set_cte(target_model.unique_id, None) return self.Relation.create_ephemeral_from_node( - self.config, target_model, sample=self.config.args.sample + self.config, target_model, empty=self.config.args.empty ) else: return self.Relation.create_from( - self.config, target_model, sample=self.config.args.sample + self.config, target_model, empty=self.config.args.empty ) def validate( @@ -594,7 +594,7 @@ def resolve(self, source_name: str, table_name: str): target_kind="source", disabled=(isinstance(target_source, Disabled)), ) - return self.Relation.create_from_source(target_source, sample=self.config.args.sample) + return self.Relation.create_from_source(target_source, empty=self.config.args.empty) # metric` implementations From 363c9b67cd4e961f2005c289701719321f18ed30 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 1 Nov 2023 16:00:30 -0400 Subject: [PATCH 03/15] safe access of self.config.args.empty --- core/dbt/context/providers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index dcaed7256e2..0dcc522369c 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -532,11 +532,11 @@ def create_relation(self, target_model: ManifestNode) -> RelationProxy: if target_model.is_ephemeral_model: self.model.set_cte(target_model.unique_id, None) return self.Relation.create_ephemeral_from_node( - self.config, target_model, empty=self.config.args.empty + self.config, target_model, empty=getattr(self.config.args, "EMPTY", False) ) else: return self.Relation.create_from( - self.config, target_model, empty=self.config.args.empty + self.config, target_model, empty=getattr(self.config.args, "EMPTY", False) ) def validate( @@ -594,7 +594,9 @@ def resolve(self, source_name: str, table_name: str): target_kind="source", disabled=(isinstance(target_source, Disabled)), ) - return self.Relation.create_from_source(target_source, empty=self.config.args.empty) + return self.Relation.create_from_source( + target_source, empty=getattr(self.config.args, "EMPTY", False) + ) # metric` implementations From 49d7a56b4d634777cf5f43ac10f23c43908d191e Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 1 Nov 2023 16:11:22 -0400 Subject: [PATCH 04/15] accept kwargs in Relation.create_ephemeral_from_node --- core/dbt/adapters/base/relation.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index b0025e18769..3aec5b74edc 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -234,13 +234,11 @@ def create_ephemeral_from_node( cls: Type[Self], config: HasQuoting, node: ManifestNode, + **kwargs: Any, ) -> Self: # Note that ephemeral models are based on the name. identifier = cls.add_ephemeral_prefix(node.name) - return cls.create( - type=cls.CTE, - identifier=identifier, - ).quote(identifier=False) + return cls.create(type=cls.CTE, identifier=identifier, **kwargs).quote(identifier=False) @classmethod def create_from_node( From b8ac4fbcfd4dc3f25d9f602853fa3550b1028b15 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 1 Nov 2023 17:48:43 -0400 Subject: [PATCH 05/15] add --no-empty option explicitly --- core/dbt/cli/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 3cea132d3a5..907f485f3b3 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -91,7 +91,7 @@ ) empty = click.option( - "--empty", + "--empty/--no-empty", envvar="DBT_EMPTY", help="If specified, limit input refs and sources to zero rows.", is_flag=True, From 094069f55c8ee87ed72ee33c7cc02b43d064bc06 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 7 Nov 2023 10:53:19 -0500 Subject: [PATCH 06/15] refactor to Relation.render_limited and BaseResolver.resolve_limit --- core/dbt/adapters/base/relation.py | 21 ++++++++++++--------- core/dbt/context/providers.py | 14 +++++++------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 3aec5b74edc..c2859a342b2 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -36,7 +36,7 @@ class BaseRelation(FakeAPIObject, Hashable): include_policy: Policy = field(default_factory=lambda: Policy()) quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False - empty: bool = False + limit: Optional[int] = None # register relation types that can be renamed for the purpose of replacing relations using stages and backups # adding a relation type here also requires defining the associated rename macro @@ -50,9 +50,6 @@ class BaseRelation(FakeAPIObject, Hashable): # include/postgres/macros/relations/view/replace.sql::postgres__get_replace_view_sql() replaceable_relations: SerializableIterable = () - # empty subquery template used when rendering a relation in the context of an --empty invocation - empty_subquery_template = "(select * from {} where false limit 0) _dbt_empty_subq" - def _is_exactish_match(self, field: ComponentName, value: str) -> bool: if self.dbt_created and self.quote_policy.get_part(field) is False: return self.path.get_lowered_part(field) == value.lower() @@ -196,10 +193,16 @@ def _render_iterator(self) -> Iterator[Tuple[Optional[ComponentName], Optional[s def render(self) -> str: # if there is nothing set, this will return the empty string. - rendered_parts = ".".join(part for _, part in self._render_iterator() if part is not None) - if self.empty and rendered_parts: - return self.empty_subquery_template.format(rendered_parts) - return rendered_parts + return ".".join(part for _, part in self._render_iterator() if part is not None) + + def render_limited(self) -> str: + rendered = self.render() + if self.limit is None: + return rendered + elif self.limit == 0: + return f"(select * from {rendered} where false limit 0) _dbt_limit_subq" + else: + return f"(select * from {rendered} limit {self.limit}) _dbt_limit_subq" def quoted(self, identifier): return "{quote_char}{identifier}{quote_char}".format( @@ -318,7 +321,7 @@ def __hash__(self) -> int: return hash(self.render()) def __str__(self) -> str: - return self.render() + return self.render() if self.limit is None else self.render_limited() @property def database(self) -> Optional[str]: diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 0dcc522369c..1bbd37af4f2 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -216,6 +216,10 @@ def current_project(self): def Relation(self): return self.db_wrapper.Relation + @property + def resolve_limit(self) -> Optional[int]: + return 0 if getattr(self.config.args, "EMPTY", False) else None + @abc.abstractmethod def __call__(self, *args: str) -> Union[str, RelationProxy, MetricReference]: pass @@ -532,12 +536,10 @@ def create_relation(self, target_model: ManifestNode) -> RelationProxy: if target_model.is_ephemeral_model: self.model.set_cte(target_model.unique_id, None) return self.Relation.create_ephemeral_from_node( - self.config, target_model, empty=getattr(self.config.args, "EMPTY", False) + self.config, target_model, limit=self.resolve_limit ) else: - return self.Relation.create_from( - self.config, target_model, empty=getattr(self.config.args, "EMPTY", False) - ) + return self.Relation.create_from(self.config, target_model, limit=self.resolve_limit) def validate( self, @@ -594,9 +596,7 @@ def resolve(self, source_name: str, table_name: str): target_kind="source", disabled=(isinstance(target_source, Disabled)), ) - return self.Relation.create_from_source( - target_source, empty=getattr(self.config.args, "EMPTY", False) - ) + return self.Relation.create_from_source(target_source, limit=self.resolve_limit) # metric` implementations From c34d699ab9a0119e61fdb01e8c4c422b4601fb23 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 16 Nov 2023 17:56:40 -0500 Subject: [PATCH 07/15] add test_render_limited unit test --- tests/unit/test_relation.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py index 94995958ba6..d8f464b473b 100644 --- a/tests/unit/test_relation.py +++ b/tests/unit/test_relation.py @@ -40,3 +40,28 @@ def test_can_be_replaced(relation_type, result): def test_can_be_replaced_default(): my_relation = BaseRelation.create(type=RelationType.View) assert my_relation.can_be_replaced is False + + +@pytest.mark.parametrize( + "limit,expected_result", + [ + (None, '"test_database"."test_schema"."test_identifier"'), + ( + 0, + '(select * from "test_database"."test_schema"."test_identifier" where false limit 0) _dbt_limit_subq', + ), + ( + 1, + '(select * from "test_database"."test_schema"."test_identifier" limit 1) _dbt_limit_subq', + ), + ], +) +def test_render_limited(limit, expected_result): + my_relation = BaseRelation.create( + database="test_database", + schema="test_schema", + identifier="test_identifier", + limit=limit, + ) + actual_result = my_relation.render_limited() + assert actual_result == expected_result From 298258a35ae9a9ac16a2a362e7df59180c0c1ca3 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 16 Nov 2023 23:33:50 -0500 Subject: [PATCH 08/15] add adapter-zone tests for --empty --- .../dbt/tests/adapter/empty/test_empty.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/adapter/dbt/tests/adapter/empty/test_empty.py diff --git a/tests/adapter/dbt/tests/adapter/empty/test_empty.py b/tests/adapter/dbt/tests/adapter/empty/test_empty.py new file mode 100644 index 00000000000..158326d49fa --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/empty/test_empty.py @@ -0,0 +1,71 @@ +import pytest +from dbt.tests.util import run_dbt, relation_from_name + + +model_input_sql = """ +select 1 as id +""" + +ephemeral_model_input_sql = """ +{{ config(materialized='ephemeral') }} +select 2 as id +""" + +raw_source_csv = """id +3 +""" + + +model_sql = """ +select * +from {{ ref('model_input') }} +union all +select * +from {{ ref('ephemeral_model_input') }} +union all +select * +from {{ source('seed_sources', 'raw_source') }} +""" + + +schema_sources_yml = """ +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_source +""" + + +class TestEmpty: + @pytest.fixture(scope="class") + def seeds(self): + return { + "raw_source.csv": raw_source_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "model_input.sql": model_input_sql, + "ephemeral_model_input.sql": ephemeral_model_input_sql, + "model.sql": model_sql, + "sources.yml": schema_sources_yml, + } + + def assert_row_count(self, project, relation_name: str, expected_row_count: int): + relation = relation_from_name(project.adapter, relation_name) + result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one") + assert result[0] == expected_row_count + + def test_run_with_empty(self, project): + # create source from seed + run_dbt(["seed"]) + + # run without empty - 3 expected rows in output - 1 from each input + run_dbt(["run"]) + self.assert_row_count(project, "model", 3) + + # run with empty - 0 expected rows in output + run_dbt(["run", "--empty"]) + self.assert_row_count(project, "model", 0) From 1cc6fdf0c6012b1dc4e456b30257a65434bfe8d5 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 16 Nov 2023 23:40:58 -0500 Subject: [PATCH 09/15] changelog entry --- .changes/unreleased/Features-20231116-234049.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20231116-234049.yaml diff --git a/.changes/unreleased/Features-20231116-234049.yaml b/.changes/unreleased/Features-20231116-234049.yaml new file mode 100644 index 00000000000..786c15311a4 --- /dev/null +++ b/.changes/unreleased/Features-20231116-234049.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support --empty flag for schema-only dry runs +time: 2023-11-16T23:40:49.96651-05:00 +custom: + Author: michelleark + Issue: "8971" From 6a37d28d8b58cad2c98b1df07b54c6b84ce0254f Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 16 Nov 2023 23:55:40 -0500 Subject: [PATCH 10/15] add unit test_providers --- tests/unit/test_providers.py | 28 ++++++++++++++++++++++++++++ tests/unit/test_relation.py | 1 + 2 files changed, 29 insertions(+) create mode 100644 tests/unit/test_providers.py diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py new file mode 100644 index 00000000000..1fa5cdc269a --- /dev/null +++ b/tests/unit/test_providers.py @@ -0,0 +1,28 @@ +import pytest +from unittest import mock + +from dbt.context.providers import BaseResolver + + +class TestBaseResolver: + class ResolverSubclass(BaseResolver): + def __call__(self, *args: str): + pass + + @pytest.fixture + def resolver(self): + return self.ResolverSubclass( + db_wrapper=mock.Mock(), + model=mock.Mock(), + config=mock.Mock(), + manifest=mock.Mock(), + ) + + @pytest.mark.parametrize( + "empty,expected_resolve_limit", + [(False, None), (True, 0)], + ) + def test_resolve_limit(self, resolver, empty, expected_resolve_limit): + resolver.config.args.EMPTY = empty + + assert resolver.resolve_limit == expected_resolve_limit diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py index d8f464b473b..e4d90c4b504 100644 --- a/tests/unit/test_relation.py +++ b/tests/unit/test_relation.py @@ -65,3 +65,4 @@ def test_render_limited(limit, expected_result): ) actual_result = my_relation.render_limited() assert actual_result == expected_result + assert str(my_relation) == expected_result From 05606eebadab3124a0a93df30c11b8657b42d4fe Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 17 Nov 2023 00:24:08 -0500 Subject: [PATCH 11/15] add TestRuntimeRefResolver and TestRuntimeSourceResolver unit test --- tests/unit/test_providers.py | 78 +++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 1fa5cdc269a..ca90580ba3f 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -1,7 +1,9 @@ import pytest from unittest import mock -from dbt.context.providers import BaseResolver +from dbt.adapters.base import BaseRelation +from dbt.context.providers import BaseResolver, RuntimeRefResolver, RuntimeSourceResolver +from dbt.contracts.graph.unparsed import Quoting class TestBaseResolver: @@ -26,3 +28,77 @@ def test_resolve_limit(self, resolver, empty, expected_resolve_limit): resolver.config.args.EMPTY = empty assert resolver.resolve_limit == expected_resolve_limit + + +class TestRuntimeRefResolver: + @pytest.fixture + def resolver(self): + mock_db_wrapper = mock.Mock() + mock_db_wrapper.Relation = BaseRelation + + return RuntimeRefResolver( + db_wrapper=mock_db_wrapper, + model=mock.Mock(), + config=mock.Mock(), + manifest=mock.Mock(), + ) + + @pytest.mark.parametrize( + "empty,is_ephemeral_model,expected_limit", + [ + (False, False, None), + (True, False, 0), + (False, True, None), + (True, True, 0), + ], + ) + def test_create_relation_with_empty(self, resolver, empty, is_ephemeral_model, expected_limit): + # setup resolver and input node + resolver.config.args.EMPTY = empty + mock_node = mock.Mock() + mock_node.database = "test" + mock_node.schema = "test" + mock_node.identifier = "test" + mock_node.alias = "test" + mock_node.is_ephemeral_model = is_ephemeral_model + + # create limited relation + with mock.patch("dbt.adapters.base.relation.ParsedNode", new=mock.Mock): + relation = resolver.create_relation(mock_node) + assert relation.limit == expected_limit + + +class TestRuntimeSourceResolver: + @pytest.fixture + def resolver(self): + mock_db_wrapper = mock.Mock() + mock_db_wrapper.Relation = BaseRelation + + return RuntimeSourceResolver( + db_wrapper=mock_db_wrapper, + model=mock.Mock(), + config=mock.Mock(), + manifest=mock.Mock(), + ) + + @pytest.mark.parametrize( + "empty,expected_limit", + [ + (False, None), + (True, 0), + ], + ) + def test_create_relation_with_empty(self, resolver, empty, expected_limit): + # setup resolver and input source + resolver.config.args.EMPTY = empty + + mock_source = mock.Mock() + mock_source.database = "test" + mock_source.schema = "test" + mock_source.identifier = "test" + mock_source.quoting = Quoting() + resolver.manifest.resolve_source.return_value = mock_source + + # create limited relation + relation = resolver.resolve("test", "test") + assert relation.limit == expected_limit From c211614a40901e98e7ef757255562478deb0af19 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 17 Nov 2023 11:58:12 -0500 Subject: [PATCH 12/15] refactor base empty tests for use in adapters --- tests/adapter/dbt/tests/adapter/empty/test_empty.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/adapter/dbt/tests/adapter/empty/test_empty.py b/tests/adapter/dbt/tests/adapter/empty/test_empty.py index 158326d49fa..a014a640c1f 100644 --- a/tests/adapter/dbt/tests/adapter/empty/test_empty.py +++ b/tests/adapter/dbt/tests/adapter/empty/test_empty.py @@ -37,7 +37,7 @@ """ -class TestEmpty: +class BaseTestEmpty: @pytest.fixture(scope="class") def seeds(self): return { @@ -69,3 +69,7 @@ def test_run_with_empty(self, project): # run with empty - 0 expected rows in output run_dbt(["run", "--empty"]) self.assert_row_count(project, "model", 0) + + +class TestEmpty(BaseTestEmpty): + pass From f0c161498e286733bbdc2ed5f73f54881e93e398 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 20 Nov 2023 17:47:37 -0500 Subject: [PATCH 13/15] remove --no-empty --- core/dbt/cli/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 907f485f3b3..3cea132d3a5 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -91,7 +91,7 @@ ) empty = click.option( - "--empty/--no-empty", + "--empty", envvar="DBT_EMPTY", help="If specified, limit input refs and sources to zero rows.", is_flag=True, From 8b591e04584e12fea77faa0534585746b401800f Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 21 Nov 2023 10:23:44 -0500 Subject: [PATCH 14/15] use limit in create_ephemeral_from_node --- core/dbt/adapters/base/relation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index c2859a342b2..55da847a716 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -237,11 +237,11 @@ def create_ephemeral_from_node( cls: Type[Self], config: HasQuoting, node: ManifestNode, - **kwargs: Any, + limit: Optional[int], ) -> Self: # Note that ephemeral models are based on the name. identifier = cls.add_ephemeral_prefix(node.name) - return cls.create(type=cls.CTE, identifier=identifier, **kwargs).quote(identifier=False) + return cls.create(type=cls.CTE, identifier=identifier, limit=limit).quote(identifier=False) @classmethod def create_from_node( From 7967be7bb373a3c737196bc0ebbe31ef6f4ed354 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 21 Nov 2023 10:30:51 -0500 Subject: [PATCH 15/15] add back --no-empty for retry --- core/dbt/cli/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 3cea132d3a5..907f485f3b3 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -91,7 +91,7 @@ ) empty = click.option( - "--empty", + "--empty/--no-empty", envvar="DBT_EMPTY", help="If specified, limit input refs and sources to zero rows.", is_flag=True,