Add environment variables for macro debugging flags (#1628)

core/dbt/clients/jinja.py

@@ -1,6 +1,7 @@
 import codecs
 import linecache
 import os
+import tempfile
 
 import jinja2
 import jinja2._compat
@@ -18,6 +19,34 @@
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
 
 
+def _linecache_inject(source, write):
+    if write:
+        # this is the only reliable way to accomplish this. Obviously, it's
+        # really darn noisy and will fill your temporary directory
+        tmp_file = tempfile.NamedTemporaryFile(
+            prefix='dbt-macro-compiled-',
+            suffix='.py',
+            delete=False,
+            mode='w+',
+            encoding='utf-8',
+        )
+        tmp_file.write(source)
+        filename = tmp_file.name
+    else:
+        filename = codecs.encode(os.urandom(12), 'hex').decode('ascii')
+
+    # encode, though I don't think this matters
+    filename = jinja2._compat.encode_filename(filename)
+    # put ourselves in the cache
+    linecache.cache[filename] = (
+        len(source),
+        None,
+        [line + '\n' for line in source.splitlines()],
+        filename
+    )
+    return filename
+
+
 class MacroFuzzParser(jinja2.parser.Parser):
     def parse_macro(self):
         node = jinja2.nodes.Macro(lineno=next(self.stream).lineno)
@@ -43,22 +72,16 @@ def _parse(self, source, name, filename):
 
     def _compile(self, source, filename):
         """Override jinja's compilation to stash the rendered source inside
-        the python linecache for debugging.
+        the python linecache for debugging when the appropriate environment
+        variable is set.
+
+        If the value is 'write', also write the files to disk.
+        WARNING: This can write a ton of data if you aren't careful.
         """
-        if filename == '<template>':
-            # make a better filename
-            filename = 'dbt-{}'.format(
-                codecs.encode(os.urandom(12), 'hex').decode('ascii')
-            )
-            # encode, though I don't think this matters
-            filename = jinja2._compat.encode_filename(filename)
-            # put ourselves in the cache
-            linecache.cache[filename] = (
-                len(source),
-                None,
-                [line + '\n' for line in source.splitlines()],
-                filename
-            )
+        macro_compile = os.environ.get('DBT_MACRO_DEBUGGING')
+        if filename == '<template>' and macro_compile:
+            write = macro_compile == 'write'
+            filename = _linecache_inject(source, write)
 
         return super(MacroFuzzEnvironment, self)._compile(source, filename)
 

core/dbt/context/common.py

@@ -195,6 +195,13 @@ def call(name):
     return call
 
 
+def _debug_here():
+    import sys
+    import ipdb
+    frame = sys._getframe(3)
+    ipdb.set_trace(frame)
+
+
 def _add_sql_handlers(context):
     sql_results = {}
     return dbt.utils.merge(context, {
@@ -415,6 +422,8 @@ def generate_base(model, model_dict, config, manifest, source_config,
         "target": target,
         "try_or_compiler_error": try_or_compiler_error(model)
     })
+    if os.environ.get('DBT_MACRO_DEBUGGING'):
+        context['debug'] = _debug_here
 
     return context