Skip to content

Commit

Permalink
Merge pull request #1128 from fishtown-analytics/cache-macro-only-man…
Browse files Browse the repository at this point in the history 
…ifest

cache the macro manifest
  • Loading branch information
@drewbanin
drewbanin authored Nov 20, 2018
2 parents 3bdebba + 8eded70 commit 416173a
Show file tree
Hide file tree
Showing 8 changed files with 93 additions and 50 deletions.
19 changes: 15 additions & 4 deletions dbt/loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def __init__(self, root_project, all_projects):
self.tests = {}
self.patches = {}
self.disabled = []
self.macro_manifest = None

def _load_macro_nodes(self, resource_type):
for project_name, project in self.all_projects.items():
Expand All @@ -31,8 +32,13 @@ def _load_macro_nodes(self, resource_type):
resource_type=resource_type,
))

# make a manifest with just the macros to get the context
self.macro_manifest = Manifest(macros=self.macros, nodes={}, docs={},
generated_at=timestring(), disabled=[])

def _load_sql_nodes(self, parser, resource_type, relative_dirs_attr,
**kwargs):

for project_name, project in self.all_projects.items():
nodes, disabled = parser.load_and_parse(
package_name=project_name,
Expand All @@ -42,6 +48,7 @@ def _load_sql_nodes(self, parser, resource_type, relative_dirs_attr,
relative_dirs=getattr(project, relative_dirs_attr),
resource_type=resource_type,
macros=self.macros,
macro_manifest=self.macro_manifest,
**kwargs
)
self.nodes.update(nodes)
Expand All @@ -59,7 +66,8 @@ def _load_seeds(self):
all_projects=self.all_projects,
root_dir=project.project_root,
relative_dirs=project.data_paths,
macros=self.macros
macros=self.macros,
macro_manifest=self.macro_manifest,
))

def _load_nodes(self):
Expand All @@ -70,10 +78,12 @@ def _load_nodes(self):
tags=['data'])

self.nodes.update(HookParser.load_and_parse(
self.root_project, self.all_projects, self.macros
self.root_project, self.all_projects, self.macros,
self.macro_manifest
))
self.nodes.update(ArchiveParser.load_and_parse(
self.root_project, self.all_projects, self.macros
self.root_project, self.all_projects, self.macros,
self.macro_manifest
))

self._load_seeds()
Expand All @@ -96,7 +106,8 @@ def _load_schema_tests(self):
all_projects=self.all_projects,
root_dir=project.project_root,
relative_dirs=project.source_paths,
macros=self.macros
macros=self.macros,
macro_manifest=self.macro_manifest,
)

for unique_id, test in tests.items():
Expand Down
4 changes: 3 additions & 1 deletion dbt/parser/archives.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def parse_archives_from_project(cls, config):
return archives

@classmethod
def load_and_parse(cls, root_project, all_projects, macros=None):
def load_and_parse(cls, root_project, all_projects, macros=None,
macro_manifest=None):
"""Load and parse archives in a list of projects. Returns a dict
that maps unique ids onto ParsedNodes"""

Expand Down Expand Up @@ -67,6 +68,7 @@ def load_and_parse(cls, root_project, all_projects, macros=None):
all_projects.get(archive.get('package_name')),
all_projects,
macros=macros,
macro_manifest=macro_manifest,
archive_config=archive_config)

return to_return
10 changes: 3 additions & 7 deletions dbt/parser/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
from dbt.utils import coalesce
from dbt.logger import GLOBAL_LOGGER as logger
from dbt.contracts.graph.parsed import ParsedNode
from dbt.contracts.graph.manifest import Manifest


class BaseParser(object):
Expand Down Expand Up @@ -41,7 +40,8 @@ def get_fqn(cls, path, package_project_config, extra=[]):
def parse_node(cls, node, node_path, root_project_config,
package_project_config, all_projects,
tags=None, fqn_extra=None, fqn=None, macros=None,
agate_table=None, archive_config=None, column_name=None):
agate_table=None, archive_config=None, column_name=None,
macro_manifest=None):
"""Parse a node, given an UnparsedNode and any other required information.
agate_table should be set if the node came from a seed file.
Expand Down Expand Up @@ -103,13 +103,9 @@ def parse_node(cls, node, node_path, root_project_config,
if column_name is not None:
node['column_name'] = column_name

# make a manifest with just the macros to get the context
manifest = Manifest(macros=macros, nodes={}, docs={},
generated_at=dbt.utils.timestring(), disabled=[])

parsed_node = ParsedNode(**node)
context = dbt.context.parser.generate(parsed_node, root_project_config,
manifest, config)
macro_manifest, config)

dbt.clients.jinja.get_rendered(
parsed_node.raw_sql, context, parsed_node.to_shallow_dict(),
Expand Down
10 changes: 6 additions & 4 deletions dbt/parser/base_sql.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ def get_compiled_path(cls, name, relative_path):

@classmethod
def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
relative_dirs, resource_type, tags=None, macros=None):
relative_dirs, resource_type, tags=None, macros=None,
macro_manifest=None):
"""Load and parse models in a list of directories. Returns a dict
that maps unique ids onto ParsedNodes"""

Expand Down Expand Up @@ -64,11 +65,11 @@ def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
})

return cls.parse_sql_nodes(result, root_project, all_projects, tags,
macros)
macros, macro_manifest)

@classmethod
def parse_sql_nodes(cls, nodes, root_project, projects,
tags=None, macros=None):
tags=None, macros=None, macro_manifest=None):

if tags is None:
tags = []
Expand All @@ -93,7 +94,8 @@ def parse_sql_nodes(cls, nodes, root_project, projects,
projects.get(package_name),
projects,
tags=tags,
macros=macros)
macros=macros,
macro_manifest=macro_manifest)

# Ignore disabled nodes
if not node_parsed['config']['enabled']:
Expand Down
12 changes: 8 additions & 4 deletions dbt/parser/hooks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def get_hooks(cls, all_projects, hook_type):

@classmethod
def load_and_parse_run_hook_type(cls, root_project, all_projects,
hook_type, macros=None):
hook_type, macros=None,
macro_manifest=None):

if dbt.flags.STRICT_MODE:
dbt.contracts.project.ProjectList(**all_projects)
Expand All @@ -64,11 +65,13 @@ def load_and_parse_run_hook_type(cls, root_project, all_projects,

tags = [hook_type]
hooks, _ = cls.parse_sql_nodes(result, root_project, all_projects,
tags=tags, macros=macros)
tags=tags, macros=macros,
macro_manifest=macro_manifest)
return hooks

@classmethod
def load_and_parse(cls, root_project, all_projects, macros=None):
def load_and_parse(cls, root_project, all_projects, macros=None,
macro_manifest=None):
if macros is None:
macros = {}

Expand All @@ -78,7 +81,8 @@ def load_and_parse(cls, root_project, all_projects, macros=None):
root_project,
all_projects,
hook_type,
macros=macros
macros=macros,
macro_manifest=macro_manifest
)
hook_nodes.update(project_hooks)

Expand Down
37 changes: 23 additions & 14 deletions dbt/parser/schemas.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ def build_unparsed_node(cls, model_name, package_name, test_type,

@classmethod
def build_parsed_node(cls, unparsed, model_name, test_namespace, test_type,
root_project, all_projects, macros, column_name):
root_project, all_projects, macros, macro_manifest,
column_name):
"""Given an UnparsedNode with a node type of Test and some extra
information, build a ParsedNode representing the test.
"""
Expand Down Expand Up @@ -208,12 +209,13 @@ def build_parsed_node(cls, unparsed, model_name, test_namespace, test_type,
fqn_extra=None,
fqn=fqn_override,
macros=macros,
column_name=column_name)
column_name=column_name,
macro_manifest=macro_manifest)

@classmethod
def build_node(cls, model_name, package_name, test_type, test_args,
root_dir, original_file_path, root_project, all_projects,
macros, column_name=None):
macros, macro_manifest, column_name=None):
"""From the various components that are common to both v1 and v2 schema,
build a ParsedNode representing a test case.
"""
Expand All @@ -232,7 +234,8 @@ def build_node(cls, model_name, package_name, test_type, test_args,

parsed = cls.build_parsed_node(unparsed, model_name, test_namespace,
original_test_type, root_project,
all_projects, macros, column_name)
all_projects, macros, macro_manifest,
column_name)
return parsed

@classmethod
Expand Down Expand Up @@ -272,7 +275,8 @@ def find_schema_yml(cls, package_name, root_dir, relative_dirs):

@classmethod
def parse_v1_test_yml(cls, original_file_path, test_yml, package_name,
root_project, all_projects, root_dir, macros=None):
root_project, all_projects, root_dir, macros=None,
macro_manifest=None):
"""Parse v1 yml contents, yielding parsed nodes.
A v1 yml file is laid out like this ('variables' written
Expand Down Expand Up @@ -327,13 +331,15 @@ def parse_v1_test_yml(cls, original_file_path, test_yml, package_name,
to_add = cls.build_node(
model_name, package_name, test_type, test_args,
root_dir, original_file_path,
root_project, all_projects, macros)
root_project, all_projects, macros,
macro_manifest)
if to_add is not None:
yield to_add

@classmethod
def parse_v2_yml(cls, original_file_path, test_yml, package_name,
root_project, all_projects, root_dir, macros):
root_project, all_projects, root_dir, macros,
macro_manifest):
"""Parse v2 yml contents, yielding both parsed nodes and node patches.
A v2 yml file is laid out like this ('variables' written
Expand Down Expand Up @@ -386,14 +392,14 @@ def parse_v2_yml(cls, original_file_path, test_yml, package_name,

iterator = cls.parse_model(model, package_name, root_dir,
original_file_path, root_project,
all_projects, macros)
all_projects, macros, macro_manifest)

for node_type, node in iterator:
yield node_type, node

@classmethod
def parse_model(cls, model, package_name, root_dir, path, root_project,
all_projects, macros):
all_projects, macros, macro_manifest):
"""Given an UnparsedNodeUpdate, return column info about the model
- column info (name and maybe description) as a dict
Expand Down Expand Up @@ -421,7 +427,8 @@ def parse_model(cls, model, package_name, root_dir, path, root_project,
)
node = cls.build_node(
model_name, package_name, test_type, test_args, root_dir,
path, root_project, all_projects, macros, column_name
path, root_project, all_projects, macros, macro_manifest,
column_name
)
yield 'test', node

Expand All @@ -431,7 +438,7 @@ def parse_model(cls, model, package_name, root_dir, path, root_project,
test_type, test_args = cls._build_v2_test_args(test, None)
node = cls.build_node(model_name, package_name, test_type,
test_args, root_dir, path, root_project,
all_projects, macros)
all_projects, macros, macro_manifest)
yield 'test', node

context = {'doc': dbt.context.parser.docs(model, docrefs)}
Expand All @@ -449,7 +456,7 @@ def parse_model(cls, model, package_name, root_dir, path, root_project,

@classmethod
def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
relative_dirs, macros=None):
relative_dirs, macros=None, macro_manifest=None):
if dbt.flags.STRICT_MODE:
dbt.contracts.project.ProjectList(**all_projects)
new_tests = {} # test unique ID -> ParsedNode
Expand All @@ -467,12 +474,14 @@ def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
(t.get('unique_id'), t)
for t in cls.parse_v1_test_yml(
original_file_path, test_yml, package_name,
root_project, all_projects, root_dir, macros)
root_project, all_projects, root_dir, macros,
macro_manifest)
)
elif version == 2:
v2_results = cls.parse_v2_yml(
original_file_path, test_yml, package_name,
root_project, all_projects, root_dir, macros)
root_project, all_projects, root_dir, macros,
macro_manifest)
for result_type, node in v2_results:
if result_type == 'patch':
node_patches[node.name] = node
Expand Down
6 changes: 4 additions & 2 deletions dbt/parser/seeds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ def parse_seed_file(cls, file_match, root_dir, package_name, should_parse):

@classmethod
def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
relative_dirs, tags=None, macros=None):
relative_dirs, tags=None, macros=None,
macro_manifest=None):
"""Load and parse seed files in a list of directories. Returns a dict
that maps unique ids onto ParsedNodes"""

Expand All @@ -70,7 +71,8 @@ def load_and_parse(cls, package_name, root_project, all_projects, root_dir,
parsed = cls.parse_node(node, node_path, root_project,
all_projects.get(package_name),
all_projects, tags=tags, macros=macros,
agate_table=agate_table)
agate_table=agate_table,
macro_manifest=macro_manifest)
result[node_path] = parsed

return result
Loading

0 comments on commit 416173a

Please sign in to comment.