Skip to content

Commit

Permalink
Remove trivial nodes before building subdag (#7194)
Browse files Browse the repository at this point in the history
* remove trial nodes before building subdag

* add changie

* Update graph.py

remove comment

* further optimize by sorting node search by degree

* change degree to product of in and out degree
  • Loading branch information
ttusing authored Apr 7, 2023
1 parent a9016c3 commit 7045e11
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 1 deletion.
6 changes: 6 additions & 0 deletions .changes/unreleased/Under the Hood-20230319-172824.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Improves build times for common selections by improving subgraph calculation
time: 2023-03-19T17:28:24.321555-07:00
custom:
Author: '@ttusing'
Issue: "7195"
21 changes: 20 additions & 1 deletion core/dbt/graph/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,26 @@ def get_subset_graph(self, selected: Iterable[UniqueId]) -> "Graph":
new_graph = self.graph.copy()
include_nodes = set(selected)

for node in self:
still_removing = True
while still_removing:
nodes_to_remove = list(
node
for node in new_graph
if node not in include_nodes
and (new_graph.in_degree(node) * new_graph.out_degree(node)) == 0
)
if len(nodes_to_remove) == 0:
still_removing = False
else:
new_graph.remove_nodes_from(nodes_to_remove)

# sort remaining nodes by degree
remaining_nodes = list(new_graph.nodes())
remaining_nodes.sort(
key=lambda node: new_graph.in_degree(node) * new_graph.out_degree(node)
)

for node in remaining_nodes:
if node not in include_nodes:
source_nodes = [x for x, _ in new_graph.in_edges(node)]
target_nodes = [x for _, x in new_graph.out_edges(node)]
Expand Down

0 comments on commit 7045e11

Please sign in to comment.