Schema Viewer Drawer (getredash#3291)

* Process extra column metadata for a few sql-based data sources.

* Add Table and Column metadata tables.

* Periodically update table and column schema tables in a celery task.

* Fetching schema returns data from table and column metadata tables.

* Add tests for backend changes.

* Front-end shows extra table metadata and uses new schema response.

* Delete datasource schema data when deleting a data source.

* Process and store data source schema when a data source is first created or after a migration.

* Tables should have a unique name per datasource.

* Addressing review comments.

* Update migration file for mixins.

* Appease PEP8

* Upgrade migration file for rebase.

* Cascade delete.

* Adding org_id

* Remove redundant column and table prefixes.

* Non-existing tables and columns should be filtered out on the server side not client side.

* Fetching table samples should be optional and should happen in a separate task per table.

* Allow users to force a schema refresh.

* Use updated_at to help prune old schema metadata periodically.

* Using settings.SCHEMAS_REFRESH_QUEUE

* fix for getredash#2426 test

* more stable test_interactive_new

* Closes #927, #928: Schema refresh improvements.

* Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936)

* Speed up schema fetch requests with fewer postgres queries.

* Add column metadata to Athena glue processing.

* Fix bug assuming 'metadata' exists for every table.

* Closes #939: Persisted, existing table metadata should be updated.

* Sample processing should be rate-limited.

* Add cli command for refreshing data samples.

* Schema refreshes should not overwrite column 'example' field.

* refresh_samples() should filter tables_to_sample on the datasource's id being sampled

* Correctly wrap long text in schema drawer.

Co-authored-by: Alison <github@bankofknowledge.net>

Schema Improvements Part 2: Add data source config options.

Adding BigQuery schema drawer with data types and samples.

client/app/assets/less/ant.less

@@ -14,6 +14,7 @@
 @import '~antd/lib/radio/style/index';
 @import '~antd/lib/time-picker/style/index';
 @import '~antd/lib/pagination/style/index';
+@import '~antd/lib/drawer/style/index';
 @import '~antd/lib/table/style/index';
 @import '~antd/lib/popover/style/index';
 @import '~antd/lib/icon/style/index';

client/app/assets/less/inc/base.less

@@ -124,6 +124,10 @@ strong {
     transition: height 0s, width 0s !important;
 }
 
+.admin-schema-editor {
+  padding: 50px 0;
+}
+
 // Ace Editor
 .ace_editor {
     border: 1px solid fade(@redash-gray, 15%) !important;

client/app/assets/less/inc/schema-browser.less

@@ -7,14 +7,14 @@ div.table-name {
   border-radius: @redash-radius;
   position: relative;
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     display: none;
   }
 
   &:hover {
     background: fade(@redash-gray, 10%);
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: flex;
     }
   }
@@ -36,7 +36,7 @@ div.table-name {
     background: transparent;
   }
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     color: fade(@redash-gray, 90%);
     cursor: pointer;
     position: absolute;
@@ -49,21 +49,25 @@ div.table-name {
     justify-content: center;
   }
 
+  .info {
+    right: 20px
+  }
+
   .table-open {
     padding: 0 22px 0 26px;
     overflow: hidden;
     text-overflow: ellipsis;
     white-space: nowrap;
     position: relative;
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: none;
     }
 
     &:hover {
       background: fade(@redash-gray, 10%);
 
-      .copy-to-editor {
+      .copy-to-editor, .info {
         display: flex;
       }
     }

client/app/components/proptypes.js

@@ -11,8 +11,16 @@ export const DataSource = PropTypes.shape({
   type_name: PropTypes.string,
 });
 
+export const DataSourceMetadata = PropTypes.shape({
+  key: PropTypes.number,
+  name: PropTypes.string,
+  type: PropTypes.string,
+  example: PropTypes.string,
+  description: PropTypes.string,
+});
+
 export const Table = PropTypes.shape({
-  columns: PropTypes.arrayOf(PropTypes.string).isRequired,
+  columns: PropTypes.arrayOf(PropTypes.object).isRequired,
 });
 
 export const Schema = PropTypes.arrayOf(Table);
@@ -31,6 +39,13 @@ export const RefreshScheduleDefault = {
   until: null,
 };
 
+export const TableMetadata = PropTypes.shape({
+  key: PropTypes.number.isRequired,
+  name: PropTypes.string.isRequired,
+  description: PropTypes.string,
+  visible: PropTypes.bool.isRequired,
+});
+
 export const Field = PropTypes.shape({
   name: PropTypes.string.isRequired,
   title: PropTypes.string,

client/app/components/queries/SchemaData.jsx

@@ -0,0 +1,101 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { react2angular } from 'react2angular';
+import Drawer from 'antd/lib/drawer';
+import Table from 'antd/lib/table';
+
+import { DataSourceMetadata } from '@/components/proptypes';
+
+function textWrapRenderer(text) {
+  return (
+    <div style={{ wordWrap: 'break-word', wordBreak: 'break-all' }}>
+      {text}
+    </div>
+  );
+}
+
+class SchemaData extends React.PureComponent {
+  static propTypes = {
+    show: PropTypes.bool.isRequired,
+    onClose: PropTypes.func.isRequired,
+    tableName: PropTypes.string,
+    tableDescription: PropTypes.string,
+    tableMetadata: PropTypes.arrayOf(DataSourceMetadata),
+  };
+
+  static defaultProps = {
+    tableName: '',
+    tableDescription: '',
+    tableMetadata: [],
+  };
+
+  render() {
+    const columns = [{
+      title: 'Column Name',
+      dataIndex: 'name',
+      width: 400,
+      key: 'name',
+      render: textWrapRenderer,
+    }, {
+      title: 'Column Type',
+      dataIndex: 'type',
+      width: 400,
+      key: 'type',
+      render: textWrapRenderer,
+    }];
+
+    const hasDescription =
+      this.props.tableMetadata.some(columnMetadata => columnMetadata.description);
+
+    const hasExample =
+      this.props.tableMetadata.some(columnMetadata => columnMetadata.example);
+
+    if (hasDescription) {
+      columns.push({
+        title: 'Description',
+        dataIndex: 'description',
+        width: 400,
+        key: 'description',
+        render: textWrapRenderer,
+      });
+    }
+
+    if (hasExample) {
+      columns.push({
+        title: 'Example',
+        dataIndex: 'example',
+        width: 400,
+        key: 'example',
+        render: textWrapRenderer,
+      });
+    }
+
+    return (
+      <Drawer
+        title={this.props.tableName}
+        closable={false}
+        placement="bottom"
+        height={500}
+        onClose={this.props.onClose}
+        visible={this.props.show}
+      >
+        <h5 className="table-description">
+          {this.props.tableDescription}
+        </h5>
+        <Table
+          dataSource={this.props.tableMetadata}
+          pagination={false}
+          scroll={{ y: 350 }}
+          size="small"
+          columns={columns}
+        />
+      </Drawer>
+    );
+  }
+}
+
+export default function init(ngModule) {
+  ngModule.component('schemaData', react2angular(SchemaData, null, []));
+}
+
+init.init = true;

client/app/components/queries/schema-browser.html

@@ -19,22 +19,32 @@
   </div>
 
   <div class="schema-browser" vs-repeat vs-size="$ctrl.getSize(table)">
-    <div ng-repeat="table in $ctrl.schema | filter:$ctrl.schemaFilterObject | filter: {name: '!'+$ctrl.versionFilter}">
+    <div ng-repeat="table in $ctrl.schema | filter:$ctrl.schemaFilterObject | filter: {name: '!'+$ctrl.versionFilter} | filter:$ctrl.itemExists track by table.name">
       <div class="table-name" ng-click="$ctrl.showTable(table)">
         <i class="fa fa-table"></i>
         <strong>
           <span title="{{table.name}}">{{table.name}}</span>
           <span ng-if="table.size !== undefined"> ({{table.size}})</span>
         </strong>
+        <i ng-if="table.hasColumnMetadata" class="fa fa-question-circle info" title="More Info" aria-hidden="true"
+          ng-click="openSchemaInfo($event, table)"></i>
         <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
           ng-click="$ctrl.itemSelected($event, [table.name])"></i>
       </div>
       <div uib-collapse="table.collapsed">
-        <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column" class="table-open">{{column}}
+        <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column.key" class="table-open">
+          {{column.name}}
           <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
-            ng-click="$ctrl.itemSelected($event, [column])"></i>
+            ng-click="$ctrl.itemSelected($event, [column.name])"></i>
         </div>
       </div>
     </div>
   </div>
+  <schema-data
+    show="showSchemaInfo"
+    table-name="tableName"
+    table-description="tableDescription"
+    table-metadata="tableMetadata"
+    on-close="closeSchemaInfo"
+  ></schema-data>
 </div>

client/app/components/queries/schema-browser.js

@@ -11,6 +11,18 @@ function SchemaBrowserCtrl($rootScope, $scope) {
     $scope.$broadcast('vsRepeatTrigger');
   };
 
+  $scope.showSchemaInfo = false;
+  $scope.openSchemaInfo = ($event, table) => {
+    $scope.tableName = table.name;
+    $scope.tableDescription = table.description;
+    $scope.tableMetadata = table.columns;
+    $scope.showSchemaInfo = true;
+    $event.stopPropagation();
+  };
+  $scope.closeSchemaInfo = () => {
+    $scope.$apply(() => { $scope.showSchemaInfo = false; });
+  };
+
   this.getSize = (table) => {
     let size = 22;
 
@@ -34,6 +46,13 @@ function SchemaBrowserCtrl($rootScope, $scope) {
     }
   };
 
+  this.itemExists = (item) => {
+    if ('visible' in item) {
+      return item.visible;
+    }
+    return false;
+  };
+
   this.itemSelected = ($event, hierarchy) => {
     $rootScope.$broadcast('query-editor.command', 'paste', hierarchy.join('.'));
     $event.preventDefault();

client/app/pages/data-sources/EditDataSource.jsx

@@ -10,6 +10,7 @@ import notification from '@/services/notification';
 import PromiseRejectionError from '@/lib/promise-rejection-error';
 import LoadingState from '@/components/items-list/components/LoadingState';
 import DynamicForm from '@/components/dynamic-form/DynamicForm';
+import SchemaTable from '@/pages/data-sources/schema-table-components/SchemaTable';
 import helper from '@/components/dynamic-form/dynamicFormHelper';
 import { HelpTrigger, TYPES as HELP_TRIGGER_TYPES } from '@/components/HelpTrigger';
 
@@ -26,13 +27,23 @@ class EditDataSource extends React.Component {
     dataSource: null,
     type: null,
     loading: true,
+    schema: null,
   };
 
   componentDidMount() {
     DataSource.get({ id: $route.current.params.dataSourceId }).$promise.then((dataSource) => {
       const { type } = dataSource;
       this.setState({ dataSource });
-      DataSource.types(types => this.setState({ type: find(types, { type }), loading: false }));
+
+      const typesPromise = DataSource.types().$promise;
+      const schemaPromise = DataSource.schema({ id: $route.current.params.dataSourceId }).$promise;
+
+      typesPromise.then(types => this.setState({ type: find(types, { type }) }));
+      schemaPromise.then(data => this.setState({ schema: data.schema }));
+
+      Promise.all([typesPromise, schemaPromise]).then(() => {
+        this.setState({ loading: false });
+      });
     }).catch((error) => {
       // ANGULAR_REMOVE_ME This code is related to Angular's HTTP services
       if (error.status && error.data) {
@@ -78,6 +89,12 @@ class EditDataSource extends React.Component {
     });
   };
 
+  updateSchema = (schema, tableId, columnId) => {
+    const { dataSource } = this.state;
+    const data = { tableId, columnId, schema };
+    DataSource.updateSchema({ id: dataSource.id }, data);
+  };
+
   testConnection = (callback) => {
     const { dataSource } = this.state;
     DataSource.test({ id: dataSource.id }, (httpResponse) => {
@@ -124,6 +141,12 @@ class EditDataSource extends React.Component {
         <div className="col-md-4 col-md-offset-4 m-b-10">
           <DynamicForm {...formProps} />
         </div>
+        <div className="col-md-12 admin-schema-editor">
+          <SchemaTable
+            schema={this.state.schema}
+            updateSchema={this.updateSchema}
+          />
+        </div>
       </div>
     );
   }