Skip to content

Commit

Permalink
Fix the test
Browse files Browse the repository at this point in the history
Make sure "dbt deps" reloads the full manifest
Make sure commands that reload the dbt_project.yml properly reset the config (including adapters)
  • Loading branch information
Jacob Beck committed Aug 5, 2020
1 parent 285479c commit bf7b1ef
Show file tree
Hide file tree
Showing 7 changed files with 25 additions and 4 deletions.
4 changes: 4 additions & 0 deletions core/dbt/adapters/base/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,10 @@ def load_macro_manifest(self) -> Manifest:
self._macro_manifest_lazy = manifest
return self._macro_manifest_lazy

def clear_macro_manifest(self):
if self._macro_manifest_lazy is not None:
self._macro_manifest_lazy = None

###
# Caching methods
###
Expand Down
3 changes: 3 additions & 0 deletions core/dbt/config/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,9 @@ def load_dependencies(self) -> Mapping[str, 'RuntimeConfig']:
self.dependencies = all_projects
return self.dependencies

def clear_dependencies(self):
self.dependencies = None

def load_projects(
self, paths: Iterable[Path]
) -> Iterator[Tuple[str, 'RuntimeConfig']]:
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/parser/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,8 @@ def load(self, macro_manifest: Manifest):
old_results = self.read_parse_results()
if old_results is not None:
logger.debug('Got an acceptable cached parse result')


self.results.macros.update(macro_manifest.macros)
self.results.files.update(macro_manifest.files)

Expand Down
10 changes: 9 additions & 1 deletion core/dbt/perf_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,22 @@
from dbt.config import RuntimeConfig


def get_full_manifest(config: RuntimeConfig) -> Manifest:
def get_full_manifest(
config: RuntimeConfig,
*,
reset: bool = False,
) -> Manifest:
"""Load the full manifest, using the adapter's internal manifest if it
exists to skip parsing internal (dbt + plugins) macros a second time.
Also, make sure that we force-laod the adapter's manifest, so it gets
attached to the adapter for any methods that need it.
"""
adapter = get_adapter(config) # type: ignore
if reset:
config.clear_dependencies()
adapter.clear_macro_manifest()

internal: Manifest = adapter.load_macro_manifest()

return load_manifest(
Expand Down
6 changes: 5 additions & 1 deletion core/dbt/rpc/task_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import dbt.exceptions
import dbt.flags as flags
from dbt.adapters.factory import get_adapter, reset_adapters, register_adapter
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.rpc import (
LastParse,
Expand Down Expand Up @@ -126,6 +127,8 @@ def reload_manifest(self) -> bool:
def reload_config(self):
config = self.config.from_args(self.args)
self.config = config
reset_adapters()
register_adapter(config)
return config

def add_request(self, request_handler: TaskHandlerProtocol):
Expand Down Expand Up @@ -184,7 +187,7 @@ def set_parsing(self) -> bool:
return True

def parse_manifest(self) -> None:
self.manifest = get_full_manifest(self.config)
self.manifest = get_full_manifest(self.config, reset=True)

def set_compile_exception(self, exc, logs=List[LogMessage]) -> None:
assert self.last_parse.state == ManifestStatus.Compiling, \
Expand Down Expand Up @@ -227,6 +230,7 @@ def get_handler(
return None

task = self.rpc_task(method)

return task

def task_table(self) -> List[TaskRow]:
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/task/rpc/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def handle_request(self) -> Result:
if dumped != self.args.vars:
self.real_task.args.vars = dumped
if isinstance(self.real_task, RemoteManifestMethod):
self.real_task.manifest = get_full_manifest(self.config)
self.real_task.manifest = get_full_manifest(self.config, reset=True)

# we parsed args from the cli, so we're set on that front
return self.real_task.handle_request()
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ def default(self, obj):
if hasattr(obj, 'to_dict'):
# if we have a to_dict we should try to serialize the result of
# that!
obj = obj.to_dict()
return obj.to_dict()
return super().default(obj)


Expand Down

0 comments on commit bf7b1ef

Please sign in to comment.