Merge pull request #52 from fishtown-analytics/fix/render-seeds-in-dag

Resource type updates

data/catalog.json

@@ -1 +1 @@
-{"nodes": {}, "generated_at": "2019-05-20T15:22:52.015910Z"}
+{"nodes": {"model.jaffle_shop.customer_orders": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "customer_orders", "comment": null, "owner": "drew"}, "columns": {"customer_id": {"type": "integer", "comment": null, "index": 1, "name": "customer_id"}, "first_order": {"type": "date", "comment": null, "index": 2, "name": "first_order"}, "most_recent_order": {"type": "date", "comment": null, "index": 3, "name": "most_recent_order"}, "number_of_orders": {"type": "bigint", "comment": null, "index": 4, "name": "number_of_orders"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.customer_orders"}, "model.jaffle_shop.customer_payments": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "customer_payments", "comment": null, "owner": "drew"}, "columns": {"customer_id": {"type": "integer", "comment": null, "index": 1, "name": "customer_id"}, "total_amount": {"type": "bigint", "comment": null, "index": 2, "name": "total_amount"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.customer_payments"}, "model.jaffle_shop.dim_customers": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "dim_customers", "comment": null, "owner": "drew"}, "columns": {"customer_id": {"type": "integer", "comment": null, "index": 1, "name": "customer_id"}, "first_order": {"type": "date", "comment": null, "index": 2, "name": "first_order"}, "most_recent_order": {"type": "date", "comment": null, "index": 3, "name": "most_recent_order"}, "number_of_orders": {"type": "bigint", "comment": null, "index": 4, "name": "number_of_orders"}, "customer_lifetime_value": {"type": "bigint", "comment": null, "index": 5, "name": "customer_lifetime_value"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.dim_customers"}, "model.jaffle_shop.fct_orders": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "fct_orders", "comment": null, "owner": "drew"}, "columns": {"order_id": {"type": "integer", "comment": null, "index": 1, "name": "order_id"}, "customer_id": {"type": "integer", "comment": null, "index": 2, "name": "customer_id"}, "order_date": {"type": "date", "comment": null, "index": 3, "name": "order_date"}, "status": {"type": "text", "comment": null, "index": 4, "name": "status"}, "credit_card_amount": {"type": "bigint", "comment": null, "index": 5, "name": "credit_card_amount"}, "coupon_amount": {"type": "bigint", "comment": null, "index": 6, "name": "coupon_amount"}, "bank_transfer_amount": {"type": "bigint", "comment": null, "index": 7, "name": "bank_transfer_amount"}, "gift_card_amount": {"type": "bigint", "comment": null, "index": 8, "name": "gift_card_amount"}, "amount": {"type": "bigint", "comment": null, "index": 9, "name": "amount"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.fct_orders"}, "model.jaffle_shop.order_payments": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "order_payments", "comment": null, "owner": "drew"}, "columns": {"order_id": {"type": "integer", "comment": null, "index": 1, "name": "order_id"}, "credit_card_amount": {"type": "bigint", "comment": null, "index": 2, "name": "credit_card_amount"}, "coupon_amount": {"type": "bigint", "comment": null, "index": 3, "name": "coupon_amount"}, "bank_transfer_amount": {"type": "bigint", "comment": null, "index": 4, "name": "bank_transfer_amount"}, "gift_card_amount": {"type": "bigint", "comment": null, "index": 5, "name": "gift_card_amount"}, "total_amount": {"type": "bigint", "comment": null, "index": 6, "name": "total_amount"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.order_payments"}, "seed.jaffle_shop.raw_customers": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "raw_customers", "comment": null, "owner": "drew"}, "columns": {"id": {"type": "integer", "comment": null, "index": 1, "name": "id"}, "first_name": {"type": "text", "comment": null, "index": 2, "name": "first_name"}, "last_name": {"type": "text", "comment": null, "index": 3, "name": "last_name"}, "email": {"type": "text", "comment": null, "index": 4, "name": "email"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "seed.jaffle_shop.raw_customers"}, "seed.jaffle_shop.raw_orders": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "raw_orders", "comment": null, "owner": "drew"}, "columns": {"id": {"type": "integer", "comment": null, "index": 1, "name": "id"}, "user_id": {"type": "integer", "comment": null, "index": 2, "name": "user_id"}, "order_date": {"type": "date", "comment": null, "index": 3, "name": "order_date"}, "status": {"type": "text", "comment": null, "index": 4, "name": "status"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "seed.jaffle_shop.raw_orders"}, "seed.jaffle_shop.raw_payments": {"metadata": {"type": "BASE TABLE", "database": "analytics", "schema": "analytics", "name": "raw_payments", "comment": null, "owner": "drew"}, "columns": {"id": {"type": "integer", "comment": null, "index": 1, "name": "id"}, "order_id": {"type": "integer", "comment": null, "index": 2, "name": "order_id"}, "payment_method": {"type": "text", "comment": null, "index": 3, "name": "payment_method"}, "amount": {"type": "integer", "comment": null, "index": 4, "name": "amount"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "seed.jaffle_shop.raw_payments"}, "model.jaffle_shop.stg_customers": {"metadata": {"type": "VIEW", "database": "analytics", "schema": "analytics", "name": "stg_customers", "comment": null, "owner": "drew"}, "columns": {"customer_id": {"type": "integer", "comment": null, "index": 1, "name": "customer_id"}, "first_name": {"type": "text", "comment": null, "index": 2, "name": "first_name"}, "last_name": {"type": "text", "comment": null, "index": 3, "name": "last_name"}, "email": {"type": "text", "comment": null, "index": 4, "name": "email"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.stg_customers"}, "model.jaffle_shop.stg_orders": {"metadata": {"type": "VIEW", "database": "analytics", "schema": "analytics", "name": "stg_orders", "comment": null, "owner": "drew"}, "columns": {"order_id": {"type": "integer", "comment": null, "index": 1, "name": "order_id"}, "customer_id": {"type": "integer", "comment": null, "index": 2, "name": "customer_id"}, "order_date": {"type": "date", "comment": null, "index": 3, "name": "order_date"}, "status": {"type": "text", "comment": null, "index": 4, "name": "status"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.stg_orders"}, "model.jaffle_shop.stg_payments": {"metadata": {"type": "VIEW", "database": "analytics", "schema": "analytics", "name": "stg_payments", "comment": null, "owner": "drew"}, "columns": {"payment_id": {"type": "integer", "comment": null, "index": 1, "name": "payment_id"}, "order_id": {"type": "integer", "comment": null, "index": 2, "name": "order_id"}, "payment_method": {"type": "text", "comment": null, "index": 3, "name": "payment_method"}, "amount": {"type": "integer", "comment": null, "index": 4, "name": "amount"}}, "stats": {"has_stats": {"id": "has_stats", "label": "Has Stats?", "value": false, "description": "Indicates whether there are statistics for this table", "include": false}}, "unique_id": "model.jaffle_shop.stg_payments"}}, "generated_at": "2019-11-12T04:07:20.086782Z"}

src/app/components/column_details/column_details.html

@@ -0,0 +1,90 @@
+<div class="panel">
+    <div class="panel-body">
+        <div ng-if="_.isEmpty(model.columns)">
+            Column information is not available for this seed
+        </div>
+        <div class="table-responsive" style="max-height: 800px; overflow-y: scroll;" ng-if="!_.isEmpty(model.columns)">
+            <table class="table table-borderless table-hover">
+                <thead>
+                    <tr>
+                        <th style="background-color: white; position: sticky; top: 0; z-index: 1;">Column</th>
+                        <th style="background-color: white; position: sticky; top: 0; z-index: 1;">Type</th>
+                        <th style="background-color: white; position: sticky; top: 0; z-index: 1;">Description</th>
+                        <th style="width: 1px; background-color: white; position: sticky; top: 0; z-index: 1;">Tests</th>
+                        <th style="width: 1px; background-color: white; position: sticky; top: 0; z-index: 1;" class='text-center'>More?</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr
+                        ng-repeat-start="column in get_columns(model) track by column.index"
+                        ng-click="toggle_column_expanded(column)"
+                        class="column-row"
+                        ng-class="{'column-row-selected': column.expanded}"
+                        ng-style="{cursor: has_more_info(column) ? 'pointer' : 'auto'}">
+                        <td>
+                            <div>
+                                <span class='text-dark'>{{ column.name }}</span>
+                            </div>
+                        </td>
+                        <td>
+                            <span class='text-dark'>{{ column.type }}</p>
+                        </td>
+                        <td style="text-overflow: ellipsis; overflow-x: hidden; white-space: nowrap; max-width: 1px;">
+                            <span ng-show="!column.expanded">{{ column.description }}</span>
+                        </td>
+                        <td>
+                            <span class="text-light" ng-show="!column.expanded">
+                                <span data-toggle="tooltip" title="Unique" ng-if="has_test(column, 'U')">U</span>
+                                <span data-toggle="tooltip" title="Not Null" ng-if="has_test(column, 'N')">N</span>
+                                <span data-toggle="tooltip" title="Foreign Key" ng-if="has_test(column, 'F')">F</span>
+                                <span data-toggle="tooltip" title="Accepted Values" ng-if="has_test(column, 'A')">A</span>
+                                <span data-toggle="tooltip" title="Custom Test" ng-if="has_test(column, '+')">+</span>
+                            </span>
+                        </td>
+                        <td class='text-center'>
+                            <span class='text-light' ng-show="has_more_info(column)">
+                                <span ng-if="column.expanded">
+                                    <svg class="icn"><use xlink:href="#icn-up"></use></svg>
+                                </span>
+                                <span ng-if="!column.expanded">
+                                    <svg class="icn"><use xlink:href="#icn-right"></use></svg>
+                                </span>
+                            </span>
+                        </td>
+                    </tr>
+                    <tr ng-repeat-end ng-show="column.expanded" style="background-color: white; padding: 10px">
+                        <td colspan="5" class="column-expanded">
+                            <div style="padding: 5px 20px">
+                                <div style="margin-bottom: 15px">
+                                    <h5>Description</h5>
+                                    <span marked="column.description"></span>
+                                </div>
+
+                                <div ng-show="column.tests && column.tests.length" style="margin-bottom: 15px">
+                                    <h5>Schema Tests</h5>
+                                    <ul class="list-unstyled" style="margin-top: 2px">
+                                        <li ng-repeat="test in column.tests" ng-switch on="test.short" class='text-light'>
+                                            <span ng-switch-when="F">
+                                                Foreign Key
+                                                <span ng-if="test.fk_model.unique_id">
+                                                    <a data-ui-state="getState(test.fk_model)"
+                                                        data-ui-state-params="{unique_id: test.fk_model.unique_id}">
+                                                        to {{ test.fk_model.name }}
+                                                    </a> on <code>{{ test.fk_field }}</code>
+                                                </span>
+                                            </span>
+                                            <span ng-switch-when="P">Primary Key</span>
+                                            <span ng-switch-when="U">Unique</span>
+                                            <span ng-switch-when="N">Not Null</span>
+                                            <span ng-switch-default>{{ test.label }}</span>
+                                        </li>
+                                    </ul>
+                                </div>
+                            </div>
+                        </td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>
+    </div>
+</div>

src/app/components/column_details/column_details.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const template = require('./column_details.html');
+
+const _ = require('underscore');
+
+angular
+.module('dbt')
+.directive('columnDetails', [function() {
+    return {
+        scope: {
+            model: '=',
+        },
+        templateUrl: template,
+        link: function(scope) {
+
+            scope.has_test = function(col, test_name) {
+                var test_types = _.pluck(col.tests, 'short');
+                return test_types.indexOf(test_name) != -1;
+            }
+
+            scope.has_more_info = function(column) {
+                var tests = (column.tests || []);
+                var description = (column.description || "");
+
+                return tests.length || description.length;
+            }
+
+            scope.toggle_column_expanded = function(column) {
+                if (scope.has_more_info(column)) {
+                    column.expanded = !column.expanded
+                }
+            }
+
+            scope.getState = function(node) {
+                return 'dbt.' + node.resource_type;
+            }
+
+            scope.get_columns = function(model) {
+                var columns = _.chain(model.columns)
+                        .values()
+                        .sortBy('index')
+                        .value();
+
+                // re-number columns because index comes from the catalog, and index may not always be present
+                // this prevents errors with the view's `track by column.index`
+                _.each(columns, function(col, i) {
+                    col.index = i;
+                });
+
+                return columns;
+            }
+        }
+    }
+}]);
+

src/app/components/graph/graph-viz.js

@@ -149,11 +149,7 @@ angular
                     onClickFunction: function(event) {
                         var target = event.target || event.cyTarget
                         var unique_id = target.id();
-                        if (target.data('resource_type') == 'source') {
-                            $state.go('dbt.source', {unique_id: unique_id});
-                        } else {
-                            $state.go('dbt.model', {unique_id: unique_id});
-                        }
+                        $state.go('dbt.' + target.data('resource_type'), {unique_id: unique_id});
                     },
                     show: true,
                 },
@@ -165,11 +161,7 @@ angular
                     onClickFunction: function(event) {
                         var target = event.target || event.cyTarget
                         var unique_id = target.id();
-                        if (target.data('resource_type') == 'source') {
-                            $state.go('dbt.source', {unique_id: unique_id});
-                        } else {
-                            $state.go('dbt.model', {unique_id: unique_id});
-                        }
+                        $state.go('dbt.' + target.data('resource_type'), {unique_id: unique_id});
                         graph.hideGraph();
                     },
                     show: true,

src/app/components/index.js

@@ -5,3 +5,4 @@ require('./model_tree/model_tree.js');
 require('./model_tree/model_tree_line.js');
 require('./search/search.js');
 require('./table_details/table_details.js');
+require('./column_details/column_details.js');

src/app/components/model_tree/model_tree.html

@@ -40,17 +40,19 @@
                 <div ng-show="tree.sources.length > 0">
                     <strong>Sources</strong>
                     <ul style="display: block">
-                        <source-tree-line
+                        <model-tree-line
                             item="item"
+                            resource-type="source"
                             ng-repeat="item in tree.sources"></source-tree-line>
                     </ul>
                     <br />
                 </div>
 
-                <strong>Models</strong>
+                <strong>Projects</strong>
                 <ul>
                     <model-tree-line
                         item="item"
+                        resource-type="model"
                         ng-repeat="item in tree.project"></model-tree-line>
                 </ul>
             </div>
@@ -62,6 +64,7 @@
                 <ul>
                     <model-tree-line
                         item="item"
+                        resource-type="model"
                         ng-repeat="item in tree.database"></model-tree-line>
                 </ul>
             </div>

src/app/components/model_tree/model_tree_line.html

@@ -3,7 +3,7 @@
     <a ng-if="!item.unique_id"
        class='unselectable'
        ng-class="getClass(item)"
-       ng-click="toggle(item)">
+       ng-click="onFolderClick(item)">
         <span class="filename">
             <span class="filename-normal">
                 <svg class="icn menu-icon-on"><use xlink:href="{{ getIcon(item.type, 'on') }}"></use></svg>
@@ -18,8 +18,7 @@
        class='unselectable'
        ng-class="getClass(item)"
        ng-click="activate(item)"
-       data-nav-unique-id="{{ item.unique_id }}"
-       ui-sref='dbt.model({unique_id: item.unique_id})'>
+       data-nav-unique-id="{{ item.unique_id }}">
         <span class="filename">
             <span class="filename-normal">
                 <svg class="icn menu-icon-on"><use xlink:href="{{ getIcon(item.type, 'on') }}"></use></svg>
@@ -31,6 +30,10 @@
     </a>
 
     <ul ng-if="item.items">
-        <model-tree-line item="subitem" depth="depth + 1" ng-repeat="subitem in item.items" />
+        <model-tree-line
+                resource-type="{{ resourceType }}"
+                item="subitem"
+                depth="depth + 1"
+                ng-repeat="subitem in item.items" />
     </ul>
 </li>