From f3a97b6c08d38189254b98ab32fabf42a2309491 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 12:03:53 +0200 Subject: [PATCH 1/9] Initial refactor of ancestors/descendants selector methods --- core/dbt/graph/selector.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index f24fb049292..24072d016cb 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -329,16 +329,32 @@ def select_childrens_parents(self, selected: Set[str]) -> Set[str]: ancestors_for = self.select_children(selected) | selected return self.select_parents(ancestors_for) | ancestors_for - def select_children(self, selected: Set[str]) -> Set[str]: + @classmethod + def descendants(cls, graph, node, max_depth: int = -1): + return nx.descendants(graph, node) + + def select_children(self, + selected: Set[str], + max_depth: int = -1) -> Set[str]: descendants: Set[str] = set() for node in selected: - descendants.update(nx.descendants(self.graph, node)) + descendants.update( + Graph.descendants(self.graph, node, max_depth=max_depth) + ) return descendants - def select_parents(self, selected: Set[str]) -> Set[str]: + @classmethod + def ancestors(cls, graph, node, max_depth: int = -1): + return nx.ancestors(graph, node) + + def select_parents(self, + selected: Set[str], + max_depth: int = -1) -> Set[str]: ancestors: Set[str] = set() for node in selected: - ancestors.update(nx.ancestors(self.graph, node)) + ancestors.update( + Graph.ancestors(self.graph, node, max_depth=max_depth) + ) return ancestors def select_successors(self, selected: Set[str]) -> Set[str]: From 7c201fed81321020153e69185b94ec418654032a Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 12:10:15 +0200 Subject: [PATCH 2/9] Added internal methods from networkx library --- core/dbt/graph/selector.py | 46 +++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 24072d016cb..bd91452ca3a 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -330,12 +330,29 @@ def select_childrens_parents(self, selected: Set[str]) -> Set[str]: return self.select_parents(ancestors_for) | ancestors_for @classmethod - def descendants(cls, graph, node, max_depth: int = -1): - return nx.descendants(graph, node) + def descendants(cls, graph, node, max_depth: int = None): + """Returns all nodes reachable from `node` in `graph`. + + Parameters + ---------- + G : NetworkX DiGraph + A directed acyclic graph (DAG) + source : node in `G` + + Returns + ------- + set() + The descendants of `node` in `graph` + """ + if not graph.has_node(node): + raise nx.NetworkXError("The node %s is not in the graph." % node) + des = set( + n for n, d in nx.single_source_shortest_path_length(graph, source=node, cutoff=max_depth).items()) + return des - {node} def select_children(self, selected: Set[str], - max_depth: int = -1) -> Set[str]: + max_depth: int = None) -> Set[str]: descendants: Set[str] = set() for node in selected: descendants.update( @@ -344,12 +361,29 @@ def select_children(self, return descendants @classmethod - def ancestors(cls, graph, node, max_depth: int = -1): - return nx.ancestors(graph, node) + def ancestors(cls, graph, node, max_depth: int = None): + """Returns all nodes having a path to `node` in `graph`. + + Parameters + ---------- + graph : NetworkX DiGraph + A directed acyclic graph (DAG) + node : node in `graph` + + Returns + ------- + set() + The ancestors of `node` in `graph` + """ + if not graph.has_node(node): + raise nx.NetworkXError("The node %s is not in the graph." % node) + anc = set( + n for n, d in nx.single_source_shortest_path_length(graph, target=node, cutoff=max_depth).items()) + return anc - {node} def select_parents(self, selected: Set[str], - max_depth: int = -1) -> Set[str]: + max_depth: int = None) -> Set[str]: ancestors: Set[str] = set() for node in selected: ancestors.update( From c1da5dd513f936481350f5c714fb909e0f275393 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 14:08:22 +0200 Subject: [PATCH 3/9] Added direct child syntax to node selector --- core/dbt/graph/selector.py | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index bd91452ca3a..7c552d2d05e 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -1,4 +1,5 @@ import os +import re from enum import Enum from itertools import chain from pathlib import Path @@ -11,8 +12,8 @@ from dbt.node_types import NodeType from dbt.exceptions import RuntimeException, InternalException, warn_or_error -SELECTOR_PARENTS = '+' -SELECTOR_CHILDREN = '+' +SELECTOR_PARENTS = r'^(?P[0-9]*)\+(?P.*)$' +SELECTOR_CHILDREN = r'^(?P.*)\+(?P[0-9]*)$' SELECTOR_GLOB = '*' SELECTOR_CHILDREN_AND_ANCESTORS = '@' SELECTOR_DELIMITER = ':' @@ -36,20 +37,30 @@ class SelectionCriteria: def __init__(self, node_spec: str): self.raw = node_spec self.select_children = False + self.select_children_max_depth = None self.select_parents = False + self.select_parents_max_depth = None self.select_childrens_parents = False if node_spec.startswith(SELECTOR_CHILDREN_AND_ANCESTORS): self.select_childrens_parents = True node_spec = node_spec[1:] - if node_spec.startswith(SELECTOR_PARENTS): + print(f"'{node_spec}', '{type(node_spec)}'") + + matches = re.match(SELECTOR_PARENTS, node_spec) + print(matches.groups()) + if matches: + print('DUPA') self.select_parents = True - node_spec = node_spec[1:] + self.select_parents_max_depth = int(matches['depth']) if matches['depth'] else None + node_spec = matches['node'] - if node_spec.endswith(SELECTOR_CHILDREN): + matches = re.match(SELECTOR_CHILDREN, node_spec) + if matches: self.select_children = True - node_spec = node_spec[:-1] + self.select_children_max_depth = int(matches['depth']) if matches['depth'] else None + node_spec = matches['node'] if self.select_children and self.select_childrens_parents: raise RuntimeException( @@ -347,13 +358,15 @@ def descendants(cls, graph, node, max_depth: int = None): if not graph.has_node(node): raise nx.NetworkXError("The node %s is not in the graph." % node) des = set( - n for n, d in nx.single_source_shortest_path_length(graph, source=node, cutoff=max_depth).items()) + n for n, d in nx.single_source_shortest_path_length(G=graph, source=node, cutoff=max_depth).items()) return des - {node} def select_children(self, selected: Set[str], max_depth: int = None) -> Set[str]: + print(f"Childrens with max depth: {max_depth}") descendants: Set[str] = set() + print(f"{selected}") for node in selected: descendants.update( Graph.descendants(self.graph, node, max_depth=max_depth) @@ -377,13 +390,16 @@ def ancestors(cls, graph, node, max_depth: int = None): """ if not graph.has_node(node): raise nx.NetworkXError("The node %s is not in the graph." % node) - anc = set( - n for n, d in nx.single_source_shortest_path_length(graph, target=node, cutoff=max_depth).items()) + with nx.utils.reversed(graph): + anc = set( + n for n, d in nx.single_source_shortest_path_length(graph, source=node, cutoff=max_depth).items()) return anc - {node} def select_parents(self, selected: Set[str], max_depth: int = None) -> Set[str]: + print(f"Parents max depth: {max_depth}") + print(f"{selected}") ancestors: Set[str] = set() for node in selected: ancestors.update( @@ -404,9 +420,10 @@ def collect_models( if spec.select_childrens_parents: additional.update(self.select_childrens_parents(selected)) if spec.select_parents: - additional.update(self.select_parents(selected)) + print('D') + additional.update(self.select_parents(selected, max_depth=spec.select_parents_max_depth)) if spec.select_children: - additional.update(self.select_children(selected)) + additional.update(self.select_children(selected, max_depth=spec.select_children_max_depth)) return additional def subgraph(self, nodes: Iterable[str]) -> 'Graph': From d81961ccee80edea2cfb29c14218cfeafbe8caca Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 16:07:49 +0200 Subject: [PATCH 4/9] flake8 --- core/dbt/graph/selector.py | 81 +++++++++++++++----------------------- 1 file changed, 32 insertions(+), 49 deletions(-) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 7c552d2d05e..276911ad3a1 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -46,20 +46,18 @@ def __init__(self, node_spec: str): self.select_childrens_parents = True node_spec = node_spec[1:] - print(f"'{node_spec}', '{type(node_spec)}'") - matches = re.match(SELECTOR_PARENTS, node_spec) - print(matches.groups()) if matches: - print('DUPA') self.select_parents = True - self.select_parents_max_depth = int(matches['depth']) if matches['depth'] else None + if matches['depth']: + self.select_parents_max_depth = int(matches['depth']) node_spec = matches['node'] matches = re.match(SELECTOR_CHILDREN, node_spec) if matches: self.select_children = True - self.select_children_max_depth = int(matches['depth']) if matches['depth'] else None + if matches['depth']: + self.select_children_max_depth = int(matches['depth']) node_spec = matches['node'] if self.select_children and self.select_childrens_parents: @@ -342,64 +340,44 @@ def select_childrens_parents(self, selected: Set[str]) -> Set[str]: @classmethod def descendants(cls, graph, node, max_depth: int = None): - """Returns all nodes reachable from `node` in `graph`. - - Parameters - ---------- - G : NetworkX DiGraph - A directed acyclic graph (DAG) - source : node in `G` - - Returns - ------- - set() - The descendants of `node` in `graph` - """ + """Returns all nodes reachable from `node` in `graph`""" if not graph.has_node(node): raise nx.NetworkXError("The node %s is not in the graph." % node) des = set( - n for n, d in nx.single_source_shortest_path_length(G=graph, source=node, cutoff=max_depth).items()) + n for n, d in + nx.single_source_shortest_path_length(G=graph, + source=node, + cutoff=max_depth).items() + ) return des - {node} + @classmethod + def ancestors(cls, graph, node, max_depth: int = None): + """Returns all nodes having a path to `node` in `graph`""" + if not graph.has_node(node): + raise nx.NetworkXError("The node %s is not in the graph." % node) + with nx.utils.reversed(graph): + anc = set( + n for n, d in + nx.single_source_shortest_path_length(G=graph, + source=node, + cutoff=max_depth).items() + ) + return anc - {node} + def select_children(self, selected: Set[str], max_depth: int = None) -> Set[str]: - print(f"Childrens with max depth: {max_depth}") descendants: Set[str] = set() - print(f"{selected}") for node in selected: descendants.update( Graph.descendants(self.graph, node, max_depth=max_depth) ) return descendants - @classmethod - def ancestors(cls, graph, node, max_depth: int = None): - """Returns all nodes having a path to `node` in `graph`. - - Parameters - ---------- - graph : NetworkX DiGraph - A directed acyclic graph (DAG) - node : node in `graph` - - Returns - ------- - set() - The ancestors of `node` in `graph` - """ - if not graph.has_node(node): - raise nx.NetworkXError("The node %s is not in the graph." % node) - with nx.utils.reversed(graph): - anc = set( - n for n, d in nx.single_source_shortest_path_length(graph, source=node, cutoff=max_depth).items()) - return anc - {node} - def select_parents(self, selected: Set[str], max_depth: int = None) -> Set[str]: - print(f"Parents max depth: {max_depth}") - print(f"{selected}") ancestors: Set[str] = set() for node in selected: ancestors.update( @@ -420,10 +398,15 @@ def collect_models( if spec.select_childrens_parents: additional.update(self.select_childrens_parents(selected)) if spec.select_parents: - print('D') - additional.update(self.select_parents(selected, max_depth=spec.select_parents_max_depth)) + additional.update( + self.select_parents(selected, + max_depth=spec.select_parents_max_depth) + ) if spec.select_children: - additional.update(self.select_children(selected, max_depth=spec.select_children_max_depth)) + additional.update( + self.select_children(selected, + max_depth=spec.select_children_max_depth) + ) return additional def subgraph(self, nodes: Iterable[str]) -> 'Graph': From bbfcce1cc6f7a8241f4fab06f4115c869ebf6e0c Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 16:42:18 +0200 Subject: [PATCH 5/9] Unit tests added for direct model selection --- test/unit/test_graph_selection.py | 92 +++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/test/unit/test_graph_selection.py b/test/unit/test_graph_selection.py index e5d86abe841..d20d234c756 100644 --- a/test/unit/test_graph_selection.py +++ b/test/unit/test_graph_selection.py @@ -110,6 +110,34 @@ def test__select_children_except_in_package(self): ['b'], set(['m.X.a','m.X.c', 'm.Y.d','m.X.e','m.Y.f','m.X.g'])) + def test__select_children(self): + self.run_specs_and_assert( + self.package_graph, + ['X.c+'], + [], + set(['m.X.c', 'm.Y.f', 'm.X.g'])) + + def test__select_children_limited(self): + self.run_specs_and_assert( + self.package_graph, + ['X.a+1'], + [], + set(['m.X.a', 'm.Y.b', 'm.X.c'])) + + def test__select_parents(self): + self.run_specs_and_assert( + self.package_graph, + ['+Y.f'], + [], + set(['m.X.c', 'm.Y.f', 'm.X.a'])) + + def test__select_parents_limited(self): + self.run_specs_and_assert( + self.package_graph, + ['1+Y.f'], + [], + set(['m.X.c', 'm.Y.f'])) + def test__select_children_except_tag(self): self.run_specs_and_assert( self.package_graph, @@ -269,10 +297,12 @@ def test__select_concat_intersection_exclude_intersection_concat(self): set(['m.X.e', 'm.Y.f']) ) - def parse_spec_and_assert(self, spec, parents, children, filter_type, filter_value, childrens_parents): + def parse_spec_and_assert(self, spec, parents, parents_max_depth, children, children_max_depth, filter_type, filter_value, childrens_parents): parsed = graph_selector.SelectionCriteria(spec) self.assertEqual(parsed.select_parents, parents) + self.assertEqual(parsed.select_parents_max_depth, parents_max_depth) self.assertEqual(parsed.select_children, children) + self.assertEqual(parsed.select_children_max_depth, children_max_depth) self.assertEqual(parsed.selector_type, filter_type) self.assertEqual(parsed.selector_value, filter_value) self.assertEqual(parsed.select_childrens_parents, childrens_parents) @@ -282,37 +312,51 @@ def invalid_spec(self, spec): graph_selector.SelectionCriteria(spec) def test__spec_parsing(self): - self.parse_spec_and_assert('a', False, False, 'fqn', 'a', False) - self.parse_spec_and_assert('+a', True, False, 'fqn', 'a', False) - self.parse_spec_and_assert('a+', False, True, 'fqn', 'a', False) - self.parse_spec_and_assert('+a+', True, True, 'fqn', 'a', False) - self.parse_spec_and_assert('@a', False, False, 'fqn', 'a', True) + self.parse_spec_and_assert('a', False, None, False, None, 'fqn', 'a', False) + self.parse_spec_and_assert('+a', True, None, False, None, 'fqn', 'a', False) + self.parse_spec_and_assert('256+a', True, 256, False, None, 'fqn', 'a', False) + self.parse_spec_and_assert('a+', False, None, True, None, 'fqn', 'a', False) + self.parse_spec_and_assert('a+256', False, None, True, 256, 'fqn', 'a', False) + self.parse_spec_and_assert('+a+', True, None, True, None, 'fqn', 'a', False) + self.parse_spec_and_assert('16+a+32', True, 16, True, 32, 'fqn', 'a', False) + self.parse_spec_and_assert('@a', False, None, False, None, 'fqn', 'a', True) self.invalid_spec('@a+') - self.parse_spec_and_assert('a.b', False, False, 'fqn', 'a.b', False) - self.parse_spec_and_assert('+a.b', True, False, 'fqn', 'a.b', False) - self.parse_spec_and_assert('a.b+', False, True, 'fqn', 'a.b', False) - self.parse_spec_and_assert('+a.b+', True, True, 'fqn', 'a.b', False) - self.parse_spec_and_assert('@a.b', False, False, 'fqn', 'a.b', True) + self.parse_spec_and_assert('a.b', False, None, False, None, 'fqn', 'a.b', False) + self.parse_spec_and_assert('+a.b', True, None, False, None, 'fqn', 'a.b', False) + self.parse_spec_and_assert('256+a.b', True, 256, False, None, 'fqn', 'a.b', False) + self.parse_spec_and_assert('a.b+', False, None, True, None, 'fqn', 'a.b', False) + self.parse_spec_and_assert('a.b+256', False, None, True, 256, 'fqn', 'a.b', False) + self.parse_spec_and_assert('+a.b+', True, None, True, None, 'fqn', 'a.b', False) + self.parse_spec_and_assert('16+a.b+32', True, 16, True, 32, 'fqn', 'a.b', False) + self.parse_spec_and_assert('@a.b', False, None, False, None, 'fqn', 'a.b', True) self.invalid_spec('@a.b+') - self.parse_spec_and_assert('a.b.*', False, False, 'fqn', 'a.b.*', False) - self.parse_spec_and_assert('+a.b.*', True, False, 'fqn', 'a.b.*', False) - self.parse_spec_and_assert('a.b.*+', False, True, 'fqn', 'a.b.*', False) - self.parse_spec_and_assert('+a.b.*+', True, True, 'fqn', 'a.b.*', False) - self.parse_spec_and_assert('@a.b.*', False, False, 'fqn', 'a.b.*', True) + self.parse_spec_and_assert('a.b.*', False, None, False, None, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('+a.b.*', True, None, False, None, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('256+a.b.*', True, 256, False, None, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('a.b.*+', False, None, True, None, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('a.b.*+256', False, None, True, 256, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('+a.b.*+', True, None, True, None, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('16+a.b.*+32', True, 16, True, 32, 'fqn', 'a.b.*', False) + self.parse_spec_and_assert('@a.b.*', False, None, False, None, 'fqn', 'a.b.*', True) self.invalid_spec('@a.b*+') - self.parse_spec_and_assert('tag:a', False, False, 'tag', 'a', False) - self.parse_spec_and_assert('+tag:a', True, False, 'tag', 'a', False) - self.parse_spec_and_assert('tag:a+', False, True, 'tag', 'a', False) - self.parse_spec_and_assert('+tag:a+', True, True, 'tag', 'a', False) - self.parse_spec_and_assert('@tag:a', False, False, 'tag', 'a', True) + self.parse_spec_and_assert('tag:a', False, None, False, None, 'tag', 'a', False) + self.parse_spec_and_assert('+tag:a', True, None, False, None, 'tag', 'a', False) + self.parse_spec_and_assert('256+tag:a', True, 256, False, None, 'tag', 'a', False) + self.parse_spec_and_assert('tag:a+', False, None, True, None, 'tag', 'a', False) + self.parse_spec_and_assert('tag:a+256', False, None, True, 256, 'tag', 'a', False) + self.parse_spec_and_assert('+tag:a+', True, None, True, None, 'tag', 'a', False) + self.parse_spec_and_assert('16+tag:a+32', True, 16, True, 32, 'tag', 'a', False) + self.parse_spec_and_assert('@tag:a', False, None, False, None, 'tag', 'a', True) self.invalid_spec('@tag:a+') - self.parse_spec_and_assert('source:a', False, False, 'source', 'a', False) - self.parse_spec_and_assert('source:a+', False, True, 'source', 'a', False) - self.parse_spec_and_assert('@source:a', False, False, 'source', 'a', True) + self.parse_spec_and_assert('source:a', False, None, False, None, 'source', 'a', False) + self.parse_spec_and_assert('source:a+', False, None, True, None, 'source', 'a', False) + self.parse_spec_and_assert('source:a+1', False, None, True, 1, 'source', 'a', False) + self.parse_spec_and_assert('source:a+32', False, None, True, 32, 'source', 'a', False) + self.parse_spec_and_assert('@source:a', False, None, False, None, 'source', 'a', True) self.invalid_spec('@source:a+') def test__package_name_getter(self): From cc2be00abb221e2b5cc8a74f2ec675f3055fc667 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 16:56:39 +0200 Subject: [PATCH 6/9] Integration tests added for direct child model selection --- .../test_graph_selection.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/integration/007_graph_selection_tests/test_graph_selection.py b/test/integration/007_graph_selection_tests/test_graph_selection.py index 0777c1e6600..8e99b06b39f 100644 --- a/test/integration/007_graph_selection_tests/test_graph_selection.py +++ b/test/integration/007_graph_selection_tests/test_graph_selection.py @@ -68,6 +68,22 @@ def test__postgres__tags_and_children(self): self.assertTrue('users' in created_models) self.assert_correct_schemas() + @use_profile('postgres') + def test__postgres__tags_and_children_limited(self): + self.run_sql_file("seed.sql") + + results = self.run_dbt(['run', '--models', 'tag:base+2']) + self.assertEqual(len(results), 3) + + created_models = self.get_models_in_schema() + self.assertFalse('base_users' in created_models) + self.assertFalse('emails' in created_models) + self.assertIn('emails_alt', created_models) + self.assertIn('users_rollup', created_models) + self.assertIn('users', created_models) + self.assertNotIn('users_rollup_dependency', created_models) + self.assert_correct_schemas() + @use_profile('snowflake') def test__snowflake__specific_model(self): self.run_sql_file("seed.sql") @@ -112,6 +128,22 @@ def test__snowflake__specific_model_and_children(self): self.assertFalse('BASE_USERS' in created_models) self.assertFalse('EMAILS' in created_models) + @use_profile('postgres') + def test__postgres__specific_model_and_children_limited(self): + self.run_sql_file("seed.sql") + + results = self.run_dbt(['run', '--models', 'users+1']) + self.assertEqual(len(results), 3) + + self.assertTablesEqual("seed", "users") + self.assertTablesEqual("summary_expected", "users_rollup") + created_models = self.get_models_in_schema() + self.assertIn('emails_alt', created_models) + self.assertNotIn('base_users', created_models) + self.assertNotIn('emails', created_models) + self.assertNotIn('users_rollup_dependency', created_models) + self.assert_correct_schemas() + @use_profile('postgres') def test__postgres__specific_model_and_parents(self): self.run_sql_file("seed.sql") @@ -142,6 +174,20 @@ def test__snowflake__specific_model_and_parents(self): self.assertFalse('BASE_USERS' in created_models) self.assertFalse('EMAILS' in created_models) + @use_profile('postgres') + def test__postgres__specific_model_and_parents_limited(self): + self.run_sql_file("seed.sql") + + results = self.run_dbt(['run', '--models', '1+users_rollup']) + self.assertEqual(len(results), 2) + + self.assertTablesEqual("seed", "users") + self.assertTablesEqual("summary_expected", "users_rollup") + created_models = self.get_models_in_schema() + self.assertFalse('base_users' in created_models) + self.assertFalse('emails' in created_models) + self.assert_correct_schemas() + @use_profile('postgres') def test__postgres__specific_model_with_exclusion(self): self.run_sql_file("seed.sql") From ce5ca6f8459ba6a1b64aa0105ad6259f8250f72a Mon Sep 17 00:00:00 2001 From: Raalsky Date: Sun, 24 May 2020 17:05:06 +0200 Subject: [PATCH 7/9] Updated CHANGELOG --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5912e917634..846673591c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,11 @@ ### Features - Added a `full_refresh` config item that overrides the behavior of the `--full-refresh` flag ([#1009](https://github.com/fishtown-analytics/dbt/issues/1009), [#2348](https://github.com/fishtown-analytics/dbt/pull/2348)) - Added a "docs" field to macros, with a "show" subfield to allow for hiding macros from the documentation site ([#2430](https://github.com/fishtown-analytics/dbt/issues/2430)) -- Added intersection syntax for model selector ([#2167](https://github.com/fishtown-analytics/dbt/issues/2167), [#2417](https://github.com/fishtown-analytics/dbt/pull/2417)) +- Added intersection syntax for model selector ([#2167](https://github.com/fishtown-analytics/dbt/issues/2167), [#2417](https://github.com/fishtown-analytics/dbt/pull/2417)) +- Extends model selection syntax with at most n-th parent/children `dbt run --models 3+m1+2` ([#2052](https://github.com/fishtown-analytics/dbt/issues/2052), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) Contributors: - - [@raalsky](https://github.com/Raalsky) ([#2417](https://github.com/fishtown-analytics/dbt/pull/2417)) + - [@raalsky](https://github.com/Raalsky) ([#2417](https://github.com/fishtown-analytics/dbt/pull/2417), [#2485](https://github.com/fishtown-analytics/dbt/pull/2485)) - [@alf-mindshift](https://github.com/alf-mindshift) ([#2431](https://github.com/fishtown-analytics/dbt/pull/2431) ## dbt 0.17.0 (Release TBD) From f214ebf4e98072dc6ad9141702b757f604ccb7b5 Mon Sep 17 00:00:00 2001 From: Raalsky Date: Tue, 26 May 2020 17:55:45 +0200 Subject: [PATCH 8/9] Graph methods refactor --- core/dbt/graph/selector.py | 44 +++++++++++++++----------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 276911ad3a1..fa2c570c30a 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -338,31 +338,25 @@ def select_childrens_parents(self, selected: Set[str]) -> Set[str]: ancestors_for = self.select_children(selected) | selected return self.select_parents(ancestors_for) | ancestors_for - @classmethod - def descendants(cls, graph, node, max_depth: int = None): + def descendants(self, node, max_depth: int = None): """Returns all nodes reachable from `node` in `graph`""" - if not graph.has_node(node): - raise nx.NetworkXError("The node %s is not in the graph." % node) - des = set( - n for n, d in - nx.single_source_shortest_path_length(G=graph, - source=node, - cutoff=max_depth).items() - ) + if not self.graph.has_node(node): + raise nx.NetworkXError(f"The node {node} is not in the graph.") + des = nx.single_source_shortest_path_length(G=self.graph, + source=node, + cutoff=max_depth)\ + .keys() return des - {node} - @classmethod - def ancestors(cls, graph, node, max_depth: int = None): + def ancestors(self, node, max_depth: int = None): """Returns all nodes having a path to `node` in `graph`""" - if not graph.has_node(node): - raise nx.NetworkXError("The node %s is not in the graph." % node) - with nx.utils.reversed(graph): - anc = set( - n for n, d in - nx.single_source_shortest_path_length(G=graph, - source=node, - cutoff=max_depth).items() - ) + if not self.graph.has_node(node): + raise nx.NetworkXError(f"The node {node} is not in the graph.") + with nx.utils.reversed(self.graph): + anc = nx.single_source_shortest_path_length(G=self.graph, + source=node, + cutoff=max_depth)\ + .keys() return anc - {node} def select_children(self, @@ -370,9 +364,7 @@ def select_children(self, max_depth: int = None) -> Set[str]: descendants: Set[str] = set() for node in selected: - descendants.update( - Graph.descendants(self.graph, node, max_depth=max_depth) - ) + descendants.update(self.descendants(node, max_depth=max_depth)) return descendants def select_parents(self, @@ -380,9 +372,7 @@ def select_parents(self, max_depth: int = None) -> Set[str]: ancestors: Set[str] = set() for node in selected: - ancestors.update( - Graph.ancestors(self.graph, node, max_depth=max_depth) - ) + ancestors.update(self.ancestors(node, max_depth=max_depth)) return ancestors def select_successors(self, selected: Set[str]) -> Set[str]: From 1d298ea5cfe63e56335b3bc0f401ce17c5f5f36f Mon Sep 17 00:00:00 2001 From: Raalsky Date: Tue, 26 May 2020 19:03:29 +0200 Subject: [PATCH 9/9] Added type annotations and errors types refactored --- core/dbt/graph/selector.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index fa2c570c30a..41a1f643bef 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -338,20 +338,20 @@ def select_childrens_parents(self, selected: Set[str]) -> Set[str]: ancestors_for = self.select_children(selected) | selected return self.select_parents(ancestors_for) | ancestors_for - def descendants(self, node, max_depth: int = None): + def descendants(self, node, max_depth: Optional[int] = None) -> Set[str]: """Returns all nodes reachable from `node` in `graph`""" if not self.graph.has_node(node): - raise nx.NetworkXError(f"The node {node} is not in the graph.") + raise InternalException(f'Node {node} not found in the graph!') des = nx.single_source_shortest_path_length(G=self.graph, source=node, cutoff=max_depth)\ .keys() return des - {node} - def ancestors(self, node, max_depth: int = None): + def ancestors(self, node, max_depth: Optional[int] = None) -> Set[str]: """Returns all nodes having a path to `node` in `graph`""" if not self.graph.has_node(node): - raise nx.NetworkXError(f"The node {node} is not in the graph.") + raise InternalException(f'Node {node} not found in the graph!') with nx.utils.reversed(self.graph): anc = nx.single_source_shortest_path_length(G=self.graph, source=node, @@ -361,7 +361,7 @@ def ancestors(self, node, max_depth: int = None): def select_children(self, selected: Set[str], - max_depth: int = None) -> Set[str]: + max_depth: Optional[int] = None) -> Set[str]: descendants: Set[str] = set() for node in selected: descendants.update(self.descendants(node, max_depth=max_depth)) @@ -369,7 +369,7 @@ def select_children(self, def select_parents(self, selected: Set[str], - max_depth: int = None) -> Set[str]: + max_depth: Optional[int] = None) -> Set[str]: ancestors: Set[str] = set() for node in selected: ancestors.update(self.ancestors(node, max_depth=max_depth))