Merge pull request #1907 from fishtown-analytics/fix/no-deepcopy-mani…

…fest

Don't deep copy the manifest unless we're in single threaded mode (#1904)

core/dbt/clients/jinja.py

@@ -16,6 +16,7 @@
 import dbt.utils
 
 from dbt.clients._jinja_blocks import BlockIterator, BlockData, BlockTag
+from dbt.flags import MACRO_DEBUGGING
 
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
 
@@ -79,9 +80,8 @@ def _compile(self, source, filename):
         If the value is 'write', also write the files to disk.
         WARNING: This can write a ton of data if you aren't careful.
         """
-        macro_compile = dbt.utils.env_set_truthy('DBT_MACRO_DEBUGGING')
-        if filename == '<template>' and macro_compile:
-            write = macro_compile == 'write'
+        if filename == '<template>' and MACRO_DEBUGGING:
+            write = MACRO_DEBUGGING == 'write'
             filename = _linecache_inject(source, write)
 
         return super()._compile(source, filename)

core/dbt/flags.py

@@ -1,5 +1,6 @@
 import os
 import multiprocessing
+from typing import Optional
 # initially all flags are set to None, the on-load call of reset() will set
 # them for their first time.
 STRICT_MODE = None
@@ -11,6 +12,21 @@
 PARTIAL_PARSE = None
 
 
+def env_set_truthy(key: str) -> Optional[str]:
+    """Return the value if it was set to a "truthy" string value, or None
+    otherwise.
+    """
+    value = os.getenv(key)
+    if not value or value.lower() in ('0', 'false', 'f'):
+        return None
+    return value
+
+
+SINGLE_THREADED_WEBSERVER = env_set_truthy('DBT_SINGLE_THREADED_WEBSERVER')
+SINGLE_THREADED_HANDLER = env_set_truthy('DBT_SINGLE_THREADED_HANDLER')
+MACRO_DEBUGGING = env_set_truthy('DBT_MACRO_DEBUGGING')
+
+
 def _get_context():
     if os.name == 'posix' and os.uname().sysname.lower() != 'darwin':
         # on linux fork is available and it's fast

core/dbt/parser/util.py

@@ -4,7 +4,8 @@
 import dbt.utils
 from dbt.node_types import NodeType
 from dbt.contracts.graph.parsed import ColumnInfo
-from dbt.config import Project
+from dbt.config import RuntimeConfig
+from dbt.flags import SINGLE_THREADED_HANDLER
 
 
 def docs(node, manifest, current_project: str, column_name=None):
@@ -254,19 +255,19 @@ def process_sources(cls, manifest, current_project: str):
         return manifest
 
     @classmethod
-    def add_new_refs(cls, manifest, current_project: Project, node, macros):
-        """Given a new node that is not in the manifest, copy the manifest and
-        insert the new node into it as if it were part of regular ref
-        processing
+    def add_new_refs(cls, manifest, config: RuntimeConfig, node, macros):
+        """Given a new node that is not in the manifest, insert the new node
+        into it as if it were part of regular ref processing.
         """
-        manifest = manifest.deepcopy()
+        if config.args.single_threaded or SINGLE_THREADED_HANDLER:
+            manifest = manifest.deepcopy()
         # it's ok for macros to silently override a local project macro name
         manifest.macros.update(macros)
 
         manifest.add_nodes({node.unique_id: node})
         cls.process_sources_for_node(
-            manifest, current_project.project_name, node
+            manifest, config.project_name, node
         )
-        cls.process_refs_for_node(manifest, current_project.project_name, node)
-        cls.process_docs_for_node(manifest, current_project.project_name, node)
+        cls.process_refs_for_node(manifest, config.project_name, node)
+        cls.process_docs_for_node(manifest, config.project_name, node)
         return manifest

core/dbt/rpc/task_handler.py

@@ -37,15 +37,13 @@
     QueueTimeoutMessage,
 )
 from dbt.rpc.method import RemoteMethod
-from dbt.utils import env_set_truthy
+from dbt.flags import SINGLE_THREADED_HANDLER
+
 
 # we use this in typing only...
 from queue import Queue  # noqa
 
 
-SINGLE_THREADED_HANDLER = env_set_truthy('DBT_SINGLE_THREADED_HANDLER')
-
-
 def sigterm_handler(signum, frame):
     raise dbt.exceptions.RPCKilledException(signum)
 

core/dbt/rpc/task_manager.py

@@ -32,10 +32,7 @@
 
 # import this to make sure our timedelta encoder is registered
 from dbt import helper_types  # noqa
-from dbt.utils import env_set_truthy
-
-
-SINGLE_THREADED_WEBSERVER = env_set_truthy('DBT_SINGLE_THREADED_WEBSERVER')
+from dbt.flags import SINGLE_THREADED_WEBSERVER
 
 
 WrappedHandler = Callable[..., Dict[str, Any]]

core/dbt/task/rpc/sql_commands.py

@@ -110,7 +110,7 @@ def _get_exec_node(self):
         node = rpc_parser.parse_remote(sql, self.args.name)
         self.manifest = ParserUtils.add_new_refs(
             manifest=self.manifest,
-            current_project=self.config,
+            config=self.config,
             node=node,
             macros=macro_overrides
         )

core/dbt/utils.py

@@ -518,16 +518,6 @@ def pluralize(count, string):
         return "{} {}s".format(count, string)
 
 
-def env_set_truthy(key: str) -> Optional[str]:
-    """Return the value if it was set to a "truthy" string value, or None
-    otherwise.
-    """
-    value = os.getenv(key)
-    if not value or value.lower() in ('0', 'false', 'f'):
-        return None
-    return value
-
-
 def restrict_to(*restrictions):
     """Create the metadata for a restricted dataclass field"""
     return {'restrict': list(restrictions)}