Skip to content

Commit

Permalink
test topological sort + depenedency list
Browse files Browse the repository at this point in the history
  • Loading branch information
@drewbanin
drewbanin committed Feb 5, 2017
1 parent 20701ba commit 9997afb
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 20 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ print $$2 }' | grep '\/test_[a-zA-Z_\-\.]\+.py'`

it:
@echo "Unit test run starting..."
@time docker-compose run test tox -e unit-py27,pep8

test:
@echo "Full test run starting..."
@time docker-compose run test tox
Expand Down
141 changes: 121 additions & 20 deletions test/unit/test_graph.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,19 @@
import dbt.project
import dbt.templates
import dbt.utils
import dbt.linker

import networkx as nx

from dbt.logger import GLOBAL_LOGGER as logger
# from dbt.logger import GLOBAL_LOGGER as logger


class FakeArgs:

def __init__(self):
self.full_refresh = False


class GraphTest(unittest.TestCase):

def tearDown(self):
Expand Down Expand Up @@ -146,14 +149,20 @@ def test__two_models_simple_ref(self):
compiler.compile(limit_to=['models'])

six.assertCountEqual(self,
self.graph_result.nodes(),
[('test_models_compile', 'model_one'),
('test_models_compile', 'model_two'),])
self.graph_result.nodes(),
[
('test_models_compile', 'model_one'),
('test_models_compile', 'model_two')
])

six.assertCountEqual(self,
self.graph_result.edges(),
[(('test_models_compile', 'model_one'),
('test_models_compile', 'model_two')),])
self.graph_result.edges(),
[
(
('test_models_compile', 'model_one'),
('test_models_compile', 'model_two')
)
])

def test__model_materializations(self):
self.use_models({
Expand All @@ -167,9 +176,9 @@ def test__model_materializations(self):
"models": {
"materialized": "table",
"test_models_compile": {
"model_one": { "materialized": "table" },
"model_two": { "materialized": "view" },
"model_three": { "materialized": "ephemeral" }
"model_one": {"materialized": "table"},
"model_two": {"materialized": "view"},
"model_three": {"materialized": "ephemeral"}
}
}
}
Expand All @@ -190,7 +199,6 @@ def test__model_materializations(self):
actual = nodes[("test_models_compile", model)]["materialized"]
self.assertEquals(actual, expected)


def test__model_enabled(self):
self.use_models({
'model_one': 'select * from events',
Expand All @@ -201,18 +209,18 @@ def test__model_enabled(self):
"models": {
"materialized": "table",
"test_models_compile": {
"model_one": { "enabled": True },
"model_two": { "enabled": False },
"model_one": {"enabled": True},
"model_two": {"enabled": False},
}
}
}

compiler = self.get_compiler(self.get_project(cfg))
compiler.compile(limit_to=['models'])


six.assertCountEqual(self,
self.graph_result.nodes(), [('test_models_compile', 'model_one')])
self.graph_result.nodes(),
[('test_models_compile', 'model_one')])

six.assertCountEqual(self, self.graph_result.edges(), [])

Expand All @@ -225,14 +233,14 @@ def test__model_incremental_without_sql_where_fails(self):
"models": {
"materialized": "table",
"test_models_compile": {
"model_one": { "materialized": "incremental" },
"model_one": {"materialized": "incremental"},
}
}
}

compiler = self.get_compiler(self.get_project(cfg))

with self.assertRaises(RuntimeError) as context:
with self.assertRaises(RuntimeError):
compiler.compile(limit_to=['models'])

def test__model_incremental(self):
Expand All @@ -245,8 +253,8 @@ def test__model_incremental(self):
"test_models_compile": {
"model_one": {
"materialized": "incremental",
"sql_where": "TRUE",
"unique_key": "TRUE"
"sql_where": "created_at",
"unique_key": "id"
},
}
}
Expand All @@ -264,3 +272,96 @@ def test__model_incremental(self):
self.graph_result.node[node]['materialized'],
'incremental')

def test__topological_ordering(self):
self.use_models({
'model_1': 'select * from events',
'model_2': 'select * from {{ ref("model_1") }}',
'model_3': '''
select * from {{ ref("model_1") }}
union all
select * from {{ ref("model_2") }}
''',
'model_4': 'select * from {{ ref("model_3") }}'
})

compiler = self.get_compiler(self.get_project({}))
compiler.compile(limit_to=['models'])

six.assertCountEqual(self,
self.graph_result.nodes(),
[
('test_models_compile', 'model_1'),
('test_models_compile', 'model_2'),
('test_models_compile', 'model_3'),
('test_models_compile', 'model_4')
])

six.assertCountEqual(self,
self.graph_result.edges(),
[
(
('test_models_compile', 'model_1'),
('test_models_compile', 'model_2')
),
(
('test_models_compile', 'model_1'),
('test_models_compile', 'model_3')
),
(
('test_models_compile', 'model_2'),
('test_models_compile', 'model_3')
),
(
('test_models_compile', 'model_3'),
('test_models_compile', 'model_4')
)
])

linker = dbt.linker.Linker()
linker.graph = self.graph_result

actual_ordering = linker.as_topological_ordering()
expected_ordering = [
('test_models_compile', 'model_1'),
('test_models_compile', 'model_2'),
('test_models_compile', 'model_3'),
('test_models_compile', 'model_4')
]

self.assertEqual(actual_ordering, expected_ordering)

def test__dependency_list(self):
self.use_models({
'model_1': 'select * from events',
'model_2': 'select * from {{ ref("model_1") }}',
'model_3': '''
select * from {{ ref("model_1") }}
union all
select * from {{ ref("model_2") }}
''',
'model_4': 'select * from {{ ref("model_3") }}'
})

compiler = self.get_compiler(self.get_project({}))
compiler.compile(limit_to=['models'])

linker = dbt.linker.Linker()
linker.graph = self.graph_result

actual_dep_list = linker.as_dependency_list()
expected_dep_list = [
[
('test_models_compile', 'model_1')
],
[
('test_models_compile', 'model_2')
],
[
('test_models_compile', 'model_3')
],
[
('test_models_compile', 'model_4'),
]
]

self.assertEqual(actual_dep_list, expected_dep_list)

0 comments on commit 9997afb

Please sign in to comment.