From 70d1922eca1408be5b7118c0a8859627228c8a76 Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Wed, 1 Mar 2023 15:16:02 -0500 Subject: [PATCH 01/10] Allow selectors to be in general syntax like selector:some_selector+1,selector:other_selector --- core/dbt/config/selectors.py | 26 ++++++++------------ core/dbt/graph/cli.py | 38 +++++++++++++----------------- core/dbt/graph/selector.py | 15 ++++++++---- core/dbt/graph/selector_methods.py | 1 + core/dbt/task/list.py | 2 +- core/dbt/task/runnable.py | 2 +- 6 files changed, 40 insertions(+), 44 deletions(-) diff --git a/core/dbt/config/selectors.py b/core/dbt/config/selectors.py index e26ee01d316..28b98710bb3 100644 --- a/core/dbt/config/selectors.py +++ b/core/dbt/config/selectors.py @@ -141,33 +141,28 @@ def validate_selector_default(selector_file: SelectorFile) -> None: # good to combine the two flows into one at some point. class SelectorDict: @classmethod - def parse_dict_definition(cls, definition, selector_dict={}): + def parse_dict_definition(cls, definition): key = list(definition)[0] value = definition[key] if isinstance(value, list): new_values = [] for sel_def in value: - new_value = cls.parse_from_definition(sel_def, selector_dict=selector_dict) + new_value = cls.parse_from_definition(sel_def) new_values.append(new_value) value = new_values if key == "exclude": definition = {key: value} elif len(definition) == 1: definition = {"method": key, "value": value} - elif key == "method" and value == "selector": - sel_def = definition.get("value") - if sel_def not in selector_dict: - raise DbtSelectorsError(f"Existing selector definition for {sel_def} not found.") - return selector_dict[definition["value"]]["definition"] return definition @classmethod - def parse_a_definition(cls, def_type, definition, selector_dict={}): + def parse_a_definition(cls, def_type, definition): # this definition must be a list new_dict = {def_type: []} for sel_def in definition[def_type]: if isinstance(sel_def, dict): - sel_def = cls.parse_from_definition(sel_def, selector_dict=selector_dict) + sel_def = cls.parse_from_definition(sel_def) new_dict[def_type].append(sel_def) elif isinstance(sel_def, str): sel_def = SelectionCriteria.dict_from_single_spec(sel_def) @@ -177,17 +172,17 @@ def parse_a_definition(cls, def_type, definition, selector_dict={}): return new_dict @classmethod - def parse_from_definition(cls, definition, selector_dict={}): + def parse_from_definition(cls, definition): if isinstance(definition, str): definition = SelectionCriteria.dict_from_single_spec(definition) elif "union" in definition: - definition = cls.parse_a_definition("union", definition, selector_dict=selector_dict) + definition = cls.parse_a_definition("union", definition) elif "intersection" in definition: definition = cls.parse_a_definition( - "intersection", definition, selector_dict=selector_dict + "intersection", definition ) elif isinstance(definition, dict): - definition = cls.parse_dict_definition(definition, selector_dict=selector_dict) + definition = cls.parse_dict_definition(definition) return definition # This is the normal entrypoint of this code. Give it the @@ -198,8 +193,7 @@ def parse_from_selectors_list(cls, selectors): for selector in selectors: sel_name = selector["name"] selector_dict[sel_name] = selector - definition = cls.parse_from_definition( - selector["definition"], selector_dict=deepcopy(selector_dict) + selector_dict[sel_name]["definition"] = cls.parse_from_definition( + selector["definition"] ) - selector_dict[sel_name]["definition"] = definition return selector_dict diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index 2950e88415e..432d837ac5b 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -123,9 +123,9 @@ def _get_list_dicts(dct: Dict[str, Any], key: str) -> List[RawDefinition]: return result -def _parse_exclusions(definition, result={}) -> Optional[SelectionSpec]: +def _parse_exclusions(definition) -> Optional[SelectionSpec]: exclusions = _get_list_dicts(definition, "exclude") - parsed_exclusions = [parse_from_definition(excl, result=result) for excl in exclusions] + parsed_exclusions = [parse_from_definition(excl) for excl in exclusions] if len(parsed_exclusions) == 1: return parsed_exclusions[0] elif len(parsed_exclusions) > 1: @@ -135,7 +135,7 @@ def _parse_exclusions(definition, result={}) -> Optional[SelectionSpec]: def _parse_include_exclude_subdefs( - definitions: List[RawDefinition], result={} + definitions: List[RawDefinition] ) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]: include_parts: List[SelectionSpec] = [] diff_arg: Optional[SelectionSpec] = None @@ -149,16 +149,16 @@ def _parse_include_exclude_subdefs( f"You cannot provide multiple exclude arguments to the " f"same selector set operator:\n{yaml_sel_cfg}" ) - diff_arg = _parse_exclusions(definition, result=result) + diff_arg = _parse_exclusions(definition) else: - include_parts.append(parse_from_definition(definition, result=result)) + include_parts.append(parse_from_definition(definition)) return (include_parts, diff_arg) -def parse_union_definition(definition: Dict[str, Any], result={}) -> SelectionSpec: +def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec: union_def_parts = _get_list_dicts(definition, "union") - include, exclude = _parse_include_exclude_subdefs(union_def_parts, result=result) + include, exclude = _parse_include_exclude_subdefs(union_def_parts) union = SelectionUnion(components=include) @@ -169,9 +169,9 @@ def parse_union_definition(definition: Dict[str, Any], result={}) -> SelectionSp return SelectionDifference(components=[union, exclude], raw=definition) -def parse_intersection_definition(definition: Dict[str, Any], result={}) -> SelectionSpec: +def parse_intersection_definition(definition: Dict[str, Any]) -> SelectionSpec: intersection_def_parts = _get_list_dicts(definition, "intersection") - include, exclude = _parse_include_exclude_subdefs(intersection_def_parts, result=result) + include, exclude = _parse_include_exclude_subdefs(intersection_def_parts) intersection = SelectionIntersection(components=include) if exclude is None: @@ -181,7 +181,7 @@ def parse_intersection_definition(definition: Dict[str, Any], result={}) -> Sele return SelectionDifference(components=[intersection, exclude], raw=definition) -def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpec: +def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec: diff_arg: Optional[SelectionSpec] = None if len(definition) == 1: key = list(definition)[0] @@ -194,15 +194,10 @@ def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpe "method": key, "value": value, } - elif definition.get("method") == "selector": - sel_def = definition.get("value") - if sel_def not in result: - raise DbtValidationError(f"Existing selector definition for {sel_def} not found.") - return result[definition["value"]]["definition"] elif "method" in definition and "value" in definition: dct = definition if "exclude" in definition: - diff_arg = _parse_exclusions(definition, result=result) + diff_arg = _parse_exclusions(definition) dct = {k: v for k, v in dct.items() if k != "exclude"} else: raise DbtValidationError( @@ -220,8 +215,7 @@ def parse_dict_definition(definition: Dict[str, Any], result={}) -> SelectionSpe def parse_from_definition( definition: RawDefinition, - rootlevel=False, - result: Dict[str, Dict[str, Union[SelectionSpec, bool]]] = {}, + rootlevel=False ) -> SelectionSpec: if ( @@ -238,11 +232,11 @@ def parse_from_definition( if isinstance(definition, str): return SelectionCriteria.from_single_spec(definition) elif "union" in definition: - return parse_union_definition(definition, result=result) + return parse_union_definition(definition) elif "intersection" in definition: - return parse_intersection_definition(definition, result=result) + return parse_intersection_definition(definition) elif isinstance(definition, dict): - return parse_dict_definition(definition, result=result) + return parse_dict_definition(definition) else: raise DbtValidationError( f"Expected to find union, intersection, str or dict, instead " @@ -259,7 +253,7 @@ def parse_from_selectors_definition( result[selector.name] = { "default": selector.default, "definition": parse_from_definition( - selector.definition, rootlevel=True, result=deepcopy(result) + selector.definition, rootlevel=True ), } return result diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index fdae6327d0e..06860a6befe 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -62,8 +62,11 @@ def select_included( """Select the explicitly included nodes, using the given spec. Return the selected set of unique IDs. """ - method = self.get_method(spec.method, spec.method_arguments) - return set(method.search(included_nodes, spec.value)) + if spec.method == 'selector': + return self.get_selected(self.project.get_selector(spec.value)) + else: + method = self.get_method(spec.method, spec.method_arguments) + return set(method.search(included_nodes, spec.value)) def get_nodes_from_criteria( self, spec: SelectionCriteria @@ -273,7 +276,7 @@ def incorporate_indirect_nodes( return selected - def get_selected(self, spec: SelectionSpec) -> Set[UniqueId]: + def get_selected(self, spec: SelectionSpec, project: Any=None) -> Set[UniqueId]: """get_selected runs through the node selection process: - node selection. Based on the include/exclude sets, the set @@ -283,15 +286,19 @@ def get_selected(self, spec: SelectionSpec) -> Set[UniqueId]: - selectors can filter the nodes after all of them have been selected """ + if project is not None: + self.project = project selected_nodes, indirect_only = self.select_nodes(spec) filtered_nodes = self.filter_selection(selected_nodes) return filtered_nodes - def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue: + def get_graph_queue(self, spec: SelectionSpec, project: Any = None) -> GraphQueue: """Returns a queue over nodes in the graph that tracks progress of dependecies. """ + if project is not None: + self.project = project selected_nodes = self.get_selected(spec) selected_resources.set_selected_resources(selected_nodes) new_graph = self.full_graph.get_subset_graph(selected_nodes) diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 2c73d480dae..10e3d62f38e 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -45,6 +45,7 @@ class MethodName(StrEnum): Metric = "metric" Result = "result" SourceStatus = "source_status" + Selector = "selector" def is_selected_node(fqn: List[str], node_selector: str): diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 429e2132e8e..05e783e2d08 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -58,7 +58,7 @@ def __init__(self, args, config, manifest): def _iterate_selected_nodes(self): selector = self.get_node_selector() spec = self.get_selection_spec() - nodes = sorted(selector.get_selected(spec)) + nodes = sorted(selector.get_selected(spec, self.config)) if not nodes: warn_or_error(NoNodesSelected()) return diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 20012cef1e1..a235ac8da93 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -125,7 +125,7 @@ def defer_to_manifest(self, adapter, selected_uids: AbstractSet[str]): def get_graph_queue(self) -> GraphQueue: selector = self.get_node_selector() spec = self.get_selection_spec() - return selector.get_graph_queue(spec) + return selector.get_graph_queue(spec, self.config) def _runtime_initialize(self): self.compile_manifest() From 098c8327ee86e03cad41f42e6a7005b7df4d230a Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Wed, 1 Mar 2023 15:25:43 -0500 Subject: [PATCH 02/10] Allow --exclude to be specified in the middle of the union --- core/dbt/graph/cli.py | 54 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index 432d837ac5b..6528d68ccc6 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -33,28 +33,64 @@ def parse_union( # turn ['a b', 'c'] -> ['a', 'b', 'c'] raw_specs = itertools.chain.from_iterable(r.split(" ") for r in components) union_components: List[SelectionSpec] = [] + exclude_components: List[SelectionSpec] = [] + in_exclude = False + flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: + if in_exclude == False and raw_spec == '--exclude': + in_exclude = True + if len(union_components) == 0: + for include in DEFAULT_INCLUDES: + union_components.append( + SelectionCriteria.from_single_spec(include, indirect_selection=False) + ) + continue + intersection_components: List[SelectionSpec] = [ SelectionCriteria.from_single_spec(part, indirect_selection=indirect_selection) for part in raw_spec.split(INTERSECTION_DELIMITER) ] - union_components.append( - SelectionIntersection( - components=intersection_components, - expect_exists=expect_exists, - raw=raw_spec, - indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), + if in_exclude: + exclude_components.append( + SelectionIntersection( + components=intersection_components, + expect_exists=expect_exists, + raw=raw_spec, + indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), + ) ) - ) - return SelectionUnion( + else: + union_components.append( + SelectionIntersection( + components=intersection_components, + expect_exists=expect_exists, + raw=raw_spec, + indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), + ) + ) + res = SelectionUnion( components=union_components, expect_exists=False, raw=components, indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION), ) + if len(exclude_components) == 0: + return res + else: + return SelectionDifference( + components=[ + res, + SelectionUnion( + components=exclude_components, + expect_exists=False, + raw=components, + indirect_selection=False + ) + ] + ) def parse_union_from_default( raw: Optional[List[str]], @@ -230,7 +266,7 @@ def parse_from_definition( f"in a root level selector definition; found {keys}." ) if isinstance(definition, str): - return SelectionCriteria.from_single_spec(definition) + return parse_union_from_default(raw=definition, default=[]) elif "union" in definition: return parse_union_definition(definition) elif "intersection" in definition: From 7a0e92d5db016e42cdc126b356691f5872c971fb Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Wed, 1 Mar 2023 16:14:39 -0500 Subject: [PATCH 03/10] Fixed a couple of issues during testing --- core/dbt/graph/cli.py | 3 ++- core/dbt/graph/selector.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index 6528d68ccc6..cae31b6fad3 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -39,6 +39,7 @@ def parse_union( flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: + print(raw_spec) if in_exclude == False and raw_spec == '--exclude': in_exclude = True if len(union_components) == 0: @@ -266,7 +267,7 @@ def parse_from_definition( f"in a root level selector definition; found {keys}." ) if isinstance(definition, str): - return parse_union_from_default(raw=definition, default=[]) + return parse_union_from_default(raw=[definition], default=[]) elif "union" in definition: return parse_union_definition(definition) elif "intersection" in definition: diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 06860a6befe..bb8c2120626 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -1,4 +1,4 @@ -from typing import Set, List, Optional, Tuple +from typing import Set, List, Optional, Tuple, Any from .graph import Graph, UniqueId from .queue import GraphQueue From e267f07e2269662fc34df82e5155a011fcbf849f Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Wed, 1 Mar 2023 16:17:13 -0500 Subject: [PATCH 04/10] removed print statement --- core/dbt/graph/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index cae31b6fad3..c6cefd46105 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -39,7 +39,6 @@ def parse_union( flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: - print(raw_spec) if in_exclude == False and raw_spec == '--exclude': in_exclude = True if len(union_components) == 0: From 37fc04a4823f2d5bca9679edb67e5140fb3aeecd Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Wed, 1 Mar 2023 16:26:25 -0500 Subject: [PATCH 05/10] Added changie --- .../unreleased/Features-20230301-162551.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .changes/unreleased/Features-20230301-162551.yaml diff --git a/.changes/unreleased/Features-20230301-162551.yaml b/.changes/unreleased/Features-20230301-162551.yaml new file mode 100644 index 00000000000..726d5d0272d --- /dev/null +++ b/.changes/unreleased/Features-20230301-162551.yaml @@ -0,0 +1,21 @@ +kind: Features +body: |- + This removes the selector check when the yaml is read in and treats it like any other method. This allows the dictionary to serve as a reference and selectors to be included in the normal command line syntax. This also supports decorators on the selector:some_selector. + + I also added --exclude as a way of specifying exclude logic in a single string. + + dbt ls --select 'selector:some_selector+1 --exclude selector:some_selector' + would select the children of some_selector one level and then subsequently remove the original. + + Most of the code removes the selectors dictionary from being passed around and instead opts to pass the project to the graph/cli get_selected and get_graph_queue methods. + + Also supports syntax like + + - definition: + union: + - selector:some_other_selector+1 --exclude some_model+2 + name: some_selector +time: 2023-03-01T16:25:51.627771-05:00 +custom: + Author: acurtis-evi + Issue: "5009" From ce1bd66465ffe31f5899370fb32a7bc59bd0d192 Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Wed, 1 Mar 2023 16:54:02 -0500 Subject: [PATCH 06/10] Updated changie --- .../unreleased/Features-20230301-162551.yaml | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/.changes/unreleased/Features-20230301-162551.yaml b/.changes/unreleased/Features-20230301-162551.yaml index 726d5d0272d..131d42e169c 100644 --- a/.changes/unreleased/Features-20230301-162551.yaml +++ b/.changes/unreleased/Features-20230301-162551.yaml @@ -1,20 +1,50 @@ kind: Features body: |- - This removes the selector check when the yaml is read in and treats it like any other method. This allows the dictionary to serve as a reference and selectors to be included in the normal command line syntax. This also supports decorators on the selector:some_selector. - - I also added --exclude as a way of specifying exclude logic in a single string. - + There are some really cool features in DBT such as the ability to choose models and their descendents, graph ordered builds, and much more. However, I found that some of the syntax seemed to be lacking (particularly around the selectors yml syntax). + + The definition part of the selector yaml supports some model selection operators, but requires that the end user use a different approach to describe what can be done in short-hand on the command line. For example, on the command line, I can select the children of a model using + + ```bash + dbt ls --select some_model+ + ``` + + In the yaml, + ```yaml + - name: some_selector + description: + - some_model+ + ``` + does not work and instead + + ```yaml + - name: some_selector + description: + - method: fqn + value: some_model + children: true + ``` + must be used. + + The yaml is very useful in defining routine selections. This inconsistency with the command line forces the end user to learn two approaches. + + Furthermore, there is support from the command line to union and intersect using the --select operator. However, if one wishes to exclude, the separate --exclude operator is needed. The syntax is very straight forward for the command line approach, but again the yaml approach is different. I see value in the yaml approach. I just believe that the two should be compatible. In this PR, I propose that the --select can include --exclude as a way of embedding excluded models in a single string. + + ```bash dbt ls --select 'selector:some_selector+1 --exclude selector:some_selector' - would select the children of some_selector one level and then subsequently remove the original. - + ``` + + In addition, dbt doesn't allow selectors to be expanded using the children/parent/ancestor operators. This PR addresses that and makes selectors just like any other method. + + Syntax in the yaml is also supported + ```yaml + - definition: + union: + - selector:some_other_selector+1 --exclude some_model+2 + name: some_selector + ``` + Most of the code removes the selectors dictionary from being passed around and instead opts to pass the project to the graph/cli get_selected and get_graph_queue methods. - Also supports syntax like - - - definition: - union: - - selector:some_other_selector+1 --exclude some_model+2 - name: some_selector time: 2023-03-01T16:25:51.627771-05:00 custom: Author: acurtis-evi From 1325b81eb5c8f44a456e164c41febf3fa178913f Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Fri, 3 Mar 2023 10:17:32 -0500 Subject: [PATCH 07/10] Change --exclude => all caps NOT --- .changes/unreleased/Features-20230301-162551.yaml | 6 +++--- core/dbt/graph/cli.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.changes/unreleased/Features-20230301-162551.yaml b/.changes/unreleased/Features-20230301-162551.yaml index 131d42e169c..710f77f72bd 100644 --- a/.changes/unreleased/Features-20230301-162551.yaml +++ b/.changes/unreleased/Features-20230301-162551.yaml @@ -27,10 +27,10 @@ body: |- The yaml is very useful in defining routine selections. This inconsistency with the command line forces the end user to learn two approaches. - Furthermore, there is support from the command line to union and intersect using the --select operator. However, if one wishes to exclude, the separate --exclude operator is needed. The syntax is very straight forward for the command line approach, but again the yaml approach is different. I see value in the yaml approach. I just believe that the two should be compatible. In this PR, I propose that the --select can include --exclude as a way of embedding excluded models in a single string. + Furthermore, there is support from the command line to union and intersect using the --select operator. However, if one wishes to exclude, the separate --exclude operator is needed. The syntax is very straight forward for the command line approach, but again the yaml approach is different. I see value in the yaml approach. I just believe that the two should be compatible. In this PR, I propose that the --select can include NOT as a way of embedding excluded models in a single string. ```bash - dbt ls --select 'selector:some_selector+1 --exclude selector:some_selector' + dbt ls --select 'selector:some_selector+1 NOT selector:some_selector' ``` In addition, dbt doesn't allow selectors to be expanded using the children/parent/ancestor operators. This PR addresses that and makes selectors just like any other method. @@ -39,7 +39,7 @@ body: |- ```yaml - definition: union: - - selector:some_other_selector+1 --exclude some_model+2 + - selector:some_other_selector+1 NOT some_model+2 name: some_selector ``` diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index c6cefd46105..6924af5a406 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -39,7 +39,7 @@ def parse_union( flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: - if in_exclude == False and raw_spec == '--exclude': + if in_exclude == False and raw_spec == 'NOT': in_exclude = True if len(union_components) == 0: for include in DEFAULT_INCLUDES: From 0fd780eb712a0800966aeeec3ae24b4b47a4a287 Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Tue, 7 Mar 2023 12:41:12 -0500 Subject: [PATCH 08/10] NOT => EXCLUDE --- core/dbt/graph/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index 6924af5a406..bcd44735885 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -39,7 +39,7 @@ def parse_union( flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: - if in_exclude == False and raw_spec == 'NOT': + if in_exclude == False and raw_spec == 'EXCLUDE': in_exclude = True if len(union_components) == 0: for include in DEFAULT_INCLUDES: From 9273f86554c5c44af3a88c70b4cc37f64562b892 Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Tue, 7 Mar 2023 13:48:11 -0500 Subject: [PATCH 09/10] Support dbt run --selector arg_selector+ --select some_models --- core/dbt/config/project.py | 2 ++ core/dbt/graph/selector.py | 4 +--- core/dbt/task/runnable.py | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 8fc115e3820..39f9ee20b5d 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -680,6 +680,8 @@ def hashed_name(self): def get_selector(self, name: str) -> Union[SelectionSpec, bool]: if name not in self.selectors: + if name == 'arg_selector' and self.arg_selector: + return self.arg_selector raise DbtRuntimeError( f"Could not find selector named {name}, expected one of {list(self.selectors)}" ) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index bb8c2120626..ad07470b1fe 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -297,9 +297,7 @@ def get_graph_queue(self, spec: SelectionSpec, project: Any = None) -> GraphQueu """Returns a queue over nodes in the graph that tracks progress of dependecies. """ - if project is not None: - self.project = project - selected_nodes = self.get_selected(spec) + selected_nodes = self.get_selected(spec, project) selected_resources.set_selected_resources(selected_nodes) new_graph = self.full_graph.get_subset_graph(selected_nodes) # should we give a way here for consumers to mutate the graph? diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index a235ac8da93..6d18a7fb3c6 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -102,6 +102,10 @@ def get_selection_spec(self) -> SelectionSpec: # Doing that is actually a little tricky, so I'm punting it to a new ticket GH #6397 indirect_selection = getattr(self.args, "INDIRECT_SELECTION", "eager") + self.config.arg_selector = None + if self.selection_arg or self.exclusion_arg: + self.config.arg_selector = parse_difference(self.selection_arg, self.exclusion_arg, indirect_selection) + if self.args.selector: # use pre-defined selector (--selector) spec = self.config.get_selector(self.args.selector) From ec540e0a828025fb5c8512c4adeeb6e01f3eeccb Mon Sep 17 00:00:00 2001 From: Andy Curtis Date: Tue, 7 Mar 2023 13:50:22 -0500 Subject: [PATCH 10/10] EXCLUDE => EXCEPT --- core/dbt/graph/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/graph/cli.py b/core/dbt/graph/cli.py index bcd44735885..99863452643 100644 --- a/core/dbt/graph/cli.py +++ b/core/dbt/graph/cli.py @@ -39,7 +39,7 @@ def parse_union( flags = get_flags() # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) for raw_spec in raw_specs: - if in_exclude == False and raw_spec == 'EXCLUDE': + if in_exclude == False and raw_spec == 'EXCEPT': in_exclude = True if len(union_components) == 0: for include in DEFAULT_INCLUDES: