Write JSON manifest file to disk

dbt/adapters/bigquery/impl.py

@@ -12,7 +12,7 @@
 
 from dbt.adapters.postgres import PostgresAdapter
 from dbt.adapters.bigquery.relation import BigQueryRelation
-from dbt.contracts.connection import validate_connection
+from dbt.contracts.connection import Connection
 from dbt.logger import GLOBAL_LOGGER as logger
 
 import google.auth
@@ -168,7 +168,7 @@ def open_connection(cls, connection):
     @classmethod
     def close(cls, connection):
         if dbt.flags.STRICT_MODE:
-            validate_connection(connection)
+            Connection(**connection)
 
         connection['state'] = 'closed'
 
@@ -314,7 +314,7 @@ def execute_model(cls, profile, project_cfg, model,
 
         if flags.STRICT_MODE:
             connection = cls.get_connection(profile, model.get('name'))
-            validate_connection(connection)
+            Connection(**connection)
 
         model_name = model.get('name')
         model_schema = model.get('schema')

dbt/adapters/default/impl.py

@@ -10,7 +10,7 @@
 import dbt.schema
 import dbt.clients.agate_helper
 
-from dbt.contracts.connection import validate_connection
+from dbt.contracts.connection import Connection
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.schema import Column
 from dbt.utils import filter_null_values
@@ -464,7 +464,7 @@ def acquire_connection(cls, profile, name):
             }
 
             if dbt.flags.STRICT_MODE:
-                validate_connection(result)
+                Connection(**result)
 
             return cls.open_connection(result)
         finally:
@@ -542,7 +542,7 @@ def begin(cls, profile, name='master'):
         connection = cls.get_connection(profile, name)
 
         if dbt.flags.STRICT_MODE:
-            validate_connection(connection)
+            Connection(**connection)
 
         if connection['transaction_open'] is True:
             raise dbt.exceptions.InternalException(
@@ -575,7 +575,7 @@ def commit(cls, profile, connection):
         global connections_in_use
 
         if dbt.flags.STRICT_MODE:
-            validate_connection(connection)
+            Connection(**connection)
 
         connection = cls.reload(connection)
 
@@ -595,7 +595,7 @@ def commit(cls, profile, connection):
     @classmethod
     def rollback(cls, connection):
         if dbt.flags.STRICT_MODE:
-            validate_connection(connection)
+            Connection(**connection)
 
         connection = cls.reload(connection)
 
@@ -615,7 +615,7 @@ def rollback(cls, connection):
     @classmethod
     def close(cls, connection):
         if dbt.flags.STRICT_MODE:
-            validate_connection(connection)
+            Connection(**connection)
 
         connection.get('handle').close()
         connection['state'] = 'closed'

dbt/api/object.py

@@ -70,8 +70,8 @@ def validate(self):
 
         for error in validator.iter_errors(self.serialize()):
             errors.append('.'.join(
-                list(map(str, error.path)) + [error.message])
-            )
+                list(map(str, error.path)) + [error.message]
+            ))
 
         if errors:
             raise ValidationException(
@@ -92,7 +92,10 @@ def __len__(self):
 
     # implement this because everyone always expects it.
     def get(self, key, default=None):
-        return self._contents.get(key, default)
+        try:
+            return self[key]
+        except KeyError:
+            return default
 
     # most users of APIObject also expect the attributes to be available via
     # dot-notation because the previous implementation assigned to __dict__.

dbt/clients/git.py

@@ -22,7 +22,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False):
 
 def list_tags(cwd):
     out, err = run_cmd(cwd, ['git', 'tag', '--list'])
-    tags = set(out.decode('utf-8').strip().split("\n"))
+    tags = out.decode('utf-8').strip().split("\n")
     return tags
 
 

dbt/clients/jinja.py

@@ -65,7 +65,7 @@ def call(*args, **kwargs):
 
 
 class MaterializationExtension(jinja2.ext.Extension):
-    tags = set(['materialization'])
+    tags = ['materialization']
 
     def parse(self, parser):
         node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)

dbt/compilation.py

@@ -1,5 +1,6 @@
 import itertools
 import os
+import json
 from collections import OrderedDict, defaultdict
 import sqlparse
 
@@ -21,9 +22,11 @@
 import dbt.loader
 import dbt.parser
 
+from dbt.clients.system import write_file
 from dbt.logger import GLOBAL_LOGGER as logger
 
 graph_file_name = 'graph.gpickle'
+manifest_file_name = 'manifest.json'
 
 
 def print_compile_stats(stats):
@@ -54,16 +57,16 @@ def prepend_ctes(model, flat_graph):
 
 def recursively_prepend_ctes(model, flat_graph):
     if dbt.flags.STRICT_MODE:
-        dbt.contracts.graph.compiled.validate_node(model)
-        dbt.contracts.graph.compiled.validate(flat_graph)
+        dbt.contracts.graph.compiled.CompiledNode(**model)
+        dbt.contracts.graph.compiled.CompiledGraph(**flat_graph)
 
     model = model.copy()
     prepend_ctes = OrderedDict()
 
     if model.get('all_ctes_injected') is True:
         return (model, model.get('extra_ctes').keys(), flat_graph)
 
-    for cte_id in model.get('extra_ctes', {}).keys():
+    for cte_id in model.get('extra_ctes', {}):
         cte_to_add = flat_graph.get('nodes').get(cte_id)
         cte_to_add, new_prepend_ctes, flat_graph = recursively_prepend_ctes(
             cte_to_add, flat_graph)
@@ -78,7 +81,7 @@ def recursively_prepend_ctes(model, flat_graph):
     model['extra_ctes'] = prepend_ctes
     model['injected_sql'] = inject_ctes_into_sql(
         model.get('compiled_sql'),
-        model.get('extra_ctes'))
+        prepend_ctes)
 
     flat_graph['nodes'][model.get('unique_id')] = model
 
@@ -148,8 +151,7 @@ def initialize(self):
     def __write(self, build_filepath, payload):
         target_path = os.path.join(self.project['target-path'], build_filepath)
 
-        dbt.clients.system.make_directory(os.path.dirname(target_path))
-        dbt.compat.write_file(target_path, payload)
+        write_file(target_path, payload)
 
         return target_path
 
@@ -207,6 +209,15 @@ def compile_node(self, node, flat_graph):
 
         return injected_node
 
+    def write_manifest_file(self, manifest):
+        """Write the manifest file to disk.
+
+        manifest should be a ParsedManifest.
+        """
+        filename = manifest_file_name
+        manifest_path = os.path.join(self.project['target-path'], filename)
+        write_file(manifest_path, json.dumps(manifest.serialize()))
+
     def write_graph_file(self, linker):
         filename = graph_file_name
         graph_path = os.path.join(self.project['target-path'], filename)
@@ -257,7 +268,7 @@ def get_all_projects(self):
             all_projects[name] = project.cfg
 
         if dbt.flags.STRICT_MODE:
-            dbt.contracts.project.validate_list(all_projects)
+            dbt.contracts.project.ProjectList(**all_projects)
 
         return all_projects
 
@@ -283,8 +294,11 @@ def compile(self):
         root_project = self.project.cfg
         all_projects = self.get_all_projects()
 
-        flat_graph = dbt.loader.GraphLoader.load_all(
-            root_project, all_projects)
+        manifest = dbt.loader.GraphLoader.load_all(root_project, all_projects)
+
+        self.write_manifest_file(manifest)
+
+        flat_graph = manifest.to_flat_graph()
 
         self._check_resource_uniqueness(flat_graph)
 

dbt/context/common.py

@@ -1,7 +1,6 @@
 import json
 import os
 import pytz
-import voluptuous
 
 from dbt.adapters.factory import get_adapter
 from dbt.compat import basestring, to_string
@@ -79,7 +78,7 @@ def _add_macros(context, model, flat_graph):
         package_name = macro.get('package_name')
 
         macro_map = {
-            macro.get('name'): macro.get('generator')(context)
+            macro.get('name'): macro.generator(context)
         }
 
         if context.get(package_name) is None:
@@ -117,9 +116,20 @@ def _add_tracking(context):
 
 
 def _add_validation(context):
+    def validate_any(*args):
+        def inner(value):
+            for arg in args:
+                if isinstance(arg, type) and isinstance(value, arg):
+                    return
+                elif value == arg:
+                    return
+            raise dbt.exceptions.ValidationException(
+                'Expected value "{}" to be one of {}'
+                .format(value, ','.join(map(str, args))))
+        return inner
+
     validation_utils = dbt.utils.AttrDict({
-        'any': voluptuous.Any,
-        'all': voluptuous.All,
+        'any': validate_any,
     })
 
     return dbt.utils.merge(

dbt/context/parser.py

@@ -10,7 +10,7 @@ def ref(db_wrapper, model, project_cfg, profile, flat_graph):
 
     def ref(*args):
         if len(args) == 1 or len(args) == 2:
-            model['refs'].append(args)
+            model['refs'].append(list(args))
 
         else:
             dbt.exceptions.ref_invalid_args(model, args)