Filter tables from schema browser (re #31)

mozilla · Jun 4, 2020 · 5465195 · 5465195
1 parent d3195f4
commit 5465195
Show file tree

Hide file tree

Showing 32 changed files with 253 additions and 11 deletions.
diff --git a/client/app/pages/queries/QuerySource.jsx b/client/app/pages/queries/QuerySource.jsx
@@ -223,6 +223,7 @@ function QuerySource(props) {
             <div className="editor__left__schema">
               <SchemaBrowser
                 schema={schema}
+                dataSourceId={dataSource ? dataSource.id : undefined}
                 onRefresh={() => refreshSchema(true)}
                 onItemSelect={handleSchemaItemSelect}
               />

diff --git a/client/app/pages/queries/components/SchemaBrowser.jsx b/client/app/pages/queries/components/SchemaBrowser.jsx
@@ -1,12 +1,16 @@
 import { isNil, map, filter, some, includes } from "lodash";
+import { axios } from "@/services/axios";
 import React, { useState, useCallback, useMemo, useEffect } from "react";
 import PropTypes from "prop-types";
 import { useDebouncedCallback } from "use-debounce";
+import Checkbox from "antd/lib/checkbox";
 import Input from "antd/lib/input";
 import Button from "antd/lib/button";
 import Tooltip from "antd/lib/tooltip";
 import AutoSizer from "react-virtualized/dist/commonjs/AutoSizer";
 import List from "react-virtualized/dist/commonjs/List";
+import { clientConfig } from "@/services/auth";
+import notification from "@/services/notification";
 
 const SchemaItemType = PropTypes.shape({
   name: PropTypes.string.isRequired,
@@ -77,9 +81,22 @@ SchemaItem.defaultProps = {
   onSelect: () => {},
 };
 
-function applyFilter(schema, filterString) {
+function applyFilter(schema, filterString, showHidden, toggleString) {
   const filters = filter(filterString.toLowerCase().split(/\s+/), s => s.length > 0);
 
+  // Filter out extra schema that match the provided toggle string
+  if (!showHidden && toggleString) {
+    const toggleStringRegex = new RegExp(toggleString);
+    try {
+        schema = filter(
+          schema,
+          item => !item.name.toLowerCase().match(toggleStringRegex)
+        );
+      } catch (err) {
+        notification.error(`Error while matching schema items: ${err}`);
+      }
+  }
+
   // Empty string: return original schema
   if (filters.length === 0) {
     return schema;
@@ -110,17 +127,39 @@ function applyFilter(schema, filterString) {
   );
 }
 
-export default function SchemaBrowser({ schema, onRefresh, onItemSelect, ...props }) {
+
+function useToggleString(dataSourceId) {
+  const [toggleString, setToggleString] = useState("");
+  useMemo(() => {
+    if (!dataSourceId) {
+      return null;
+    }
+    axios.get(
+      `${clientConfig.basePath}api/data_sources/${dataSourceId}/toggle_string`
+    ).then(data => {
+      setToggleString(data.toggle_string);
+    })
+  }, [dataSourceId]);
+  return toggleString;
+}
+
+
+export default function SchemaBrowser({ schema, dataSourceId, onRefresh, onItemSelect, ...props }) {
   const [filterString, setFilterString] = useState("");
-  const filteredSchema = useMemo(() => applyFilter(schema, filterString), [schema, filterString]);
+  const [showHidden, setShowHidden] = useState(false);
+  const toggleString = useToggleString(dataSourceId);
+  const filteredSchema = useMemo(() => applyFilter(schema, filterString,
+    showHidden, toggleString), [schema, filterString, showHidden, toggleString]);
   const [expandedFlags, setExpandedFlags] = useState({});
   const [handleFilterChange] = useDebouncedCallback(setFilterString, 500);
+  const [handleToggleChange] = useDebouncedCallback(setShowHidden, 100);
   const [listRef, setListRef] = useState(null);
 
   useEffect(() => {
     setExpandedFlags({});
   }, [schema]);
 
+
   useEffect(() => {
     if (listRef) {
       listRef.recomputeRowHeights();
@@ -147,13 +186,21 @@ export default function SchemaBrowser({ schema, onRefresh, onItemSelect, ...prop
           disabled={schema.length === 0}
           onChange={event => handleFilterChange(event.target.value)}
         />
-
         <Tooltip title="Refresh Schema">
           <Button onClick={onRefresh}>
             <i className="zmdi zmdi-refresh" />
           </Button>
         </Tooltip>
       </div>
+      <div>
+        {toggleString && <Tooltip placement="right" title={`Matching pattern: ${toggleString}`}>
+        <Checkbox
+          className="m-t-10"
+          checked={showHidden}
+          onChange={event => handleToggleChange(event.target.checked)}>
+            Show hidden schema
+        </Checkbox></Tooltip>}
+      </div>
       <div className="schema-browser">
         <AutoSizer>
           {({ width, height }) => (
@@ -190,12 +237,14 @@ export default function SchemaBrowser({ schema, onRefresh, onItemSelect, ...prop
 
 SchemaBrowser.propTypes = {
   schema: PropTypes.arrayOf(SchemaItemType),
+  dataSourceId: PropTypes.number,
   onRefresh: PropTypes.func,
   onItemSelect: PropTypes.func,
 };
 
 SchemaBrowser.defaultProps = {
   schema: [],
+  dataSourceId: null,
   onRefresh: () => {},
   onItemSelect: () => {},
 };
diff --git a/redash/handlers/api.py b/redash/handlers/api.py
@@ -25,6 +25,7 @@
     DataSourceSchemaResource,
     DataSourceTestResource,
     DataSourceTypeListResource,
+    DataSourceToggleStringResource,
 )
 from redash.handlers.destinations import (
     DestinationListResource,
@@ -152,6 +153,8 @@ def json_representation(data, code, headers=None):
 api.add_org_resource(
     DataSourceResource, "/api/data_sources/<data_source_id>", endpoint="data_source"
 )
+api.add_resource(DataSourceToggleStringResource, "/api/data_sources/<data_source_id>/toggle_string")
+
 
 api.add_org_resource(GroupListResource, "/api/groups", endpoint="groups")
 api.add_org_resource(GroupResource, "/api/groups/<group_id>", endpoint="group")

diff --git a/redash/handlers/data_sources.py b/redash/handlers/data_sources.py
@@ -266,3 +266,17 @@ def post(self, data_source_id):
             }
         )
         return response
+
+
+class DataSourceToggleStringResource(BaseResource):
+    def get(self, data_source_id):
+        data_source = get_object_or_404(
+            models.DataSource.get_by_id_and_org, data_source_id, self.current_org
+        )
+        require_access(data_source.groups, self.current_user, view_only)
+        try:
+            return {
+                "toggle_string": data_source.options.get("toggle_table_string", "")
+            }
+        except Exception:
+            abort(400)
diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py
@@ -232,6 +232,12 @@ def configuration_schema(cls):
                 "url": {"type": "string", "title": cls.url_title},
                 "username": {"type": "string", "title": cls.username_title},
                 "password": {"type": "string", "title": cls.password_title},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "secret": ["password"],
             "order": ["url", "username", "password"],

diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py
@@ -82,6 +82,12 @@ def configuration_schema(cls):
                     "title": "Athena cost per Tb scanned (USD)",
                     "default": 5,
                 },
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["region", "s3_staging_dir"],
             "extra_options": ["glue", "cost_per_tb"],

diff --git a/redash/query_runner/axibase_tsd.py b/redash/query_runner/axibase_tsd.py
@@ -115,6 +115,12 @@ def configuration_schema(cls):
                     "type": "boolean",
                     "title": "Trust SSL Certificate",
                 },
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["username", "password", "hostname", "protocol", "port"],
             "secret": ["password"],

diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py
@@ -117,6 +117,12 @@ def configuration_schema(cls):
                     "type": "number",
                     "title": "Maximum Billing Tier",
                 },
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["jsonKeyFile", "projectId"],
             "order": [

diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py
@@ -77,6 +77,12 @@ def configuration_schema(cls):
                         "PROTOCOL_TLSv1_2",
                     ],
                 },
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["keyspace", "host", "useSsl"],
             "secret": ["sslCertificateFile"],

diff --git a/redash/query_runner/clickhouse.py b/redash/query_runner/clickhouse.py
@@ -32,6 +32,12 @@ def configuration_schema(cls):
                     "title": "Verify SSL certificate",
                     "default": True,
                 },
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "order": ["url", "user", "password", "dbname"],
             "required": ["dbname"],

diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py
@@ -44,6 +44,12 @@ def configuration_schema(cls):
                 "region": {"type": "string", "default": "us-east-1"},
                 "access_key": {"type": "string"},
                 "secret_key": {"type": "string"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["access_key", "secret_key"],
             "secret": ["secret_key"],

diff --git a/redash/query_runner/elasticsearch.py b/redash/query_runner/elasticsearch.py
@@ -56,6 +56,12 @@ def configuration_schema(cls):
                     "type": "string",
                     "title": "Basic Auth Password",
                 },
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "order": ["server", "basic_auth_user", "basic_auth_password"],
             "secret": ["basic_auth_password"],

diff --git a/redash/query_runner/google_analytics.py b/redash/query_runner/google_analytics.py
@@ -103,7 +103,15 @@ def enabled(cls):
     def configuration_schema(cls):
         return {
             "type": "object",
-            "properties": {"jsonKeyFile": {"type": "string", "title": "JSON Key File"}},
+            "properties": {
+                "jsonKeyFile": {"type": "string", "title": "JSON Key File"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
+            },
             "required": ["jsonKeyFile"],
             "secret": ["jsonKeyFile"],
         }

diff --git a/redash/query_runner/google_spreadsheets.py b/redash/query_runner/google_spreadsheets.py
@@ -166,7 +166,15 @@ def enabled(cls):
     def configuration_schema(cls):
         return {
             "type": "object",
-            "properties": {"jsonKeyFile": {"type": "string", "title": "JSON Key File"}},
+            "properties": {
+                "jsonKeyFile": {"type": "string", "title": "JSON Key File"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
+            },
             "required": ["jsonKeyFile"],
             "secret": ["jsonKeyFile"],
         }

diff --git a/redash/query_runner/graphite.py b/redash/query_runner/graphite.py
@@ -45,6 +45,12 @@ def configuration_schema(cls):
                 "username": {"type": "string"},
                 "password": {"type": "string"},
                 "verify": {"type": "boolean", "title": "Verify SSL certificate"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["url"],
             "secret": ["password"],

diff --git a/redash/query_runner/hive_ds.py b/redash/query_runner/hive_ds.py
@@ -50,6 +50,12 @@ def configuration_schema(cls):
                 "port": {"type": "number"},
                 "database": {"type": "string"},
                 "username": {"type": "string"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "order": ["host", "port", "database", "username"],
             "required": ["host"],
@@ -111,7 +117,6 @@ def _get_connection(self):
             database=self.configuration.get("database", "default"),
             username=self.configuration.get("username", None),
         )
-
         return connection
 
     def run_query(self, query, user):

diff --git a/redash/query_runner/impala_ds.py b/redash/query_runner/impala_ds.py
@@ -56,6 +56,12 @@ def configuration_schema(cls):
                 "ldap_user": {"type": "string"},
                 "ldap_password": {"type": "string"},
                 "timeout": {"type": "number"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["host"],
             "secret": ["ldap_password"],

diff --git a/redash/query_runner/influx_db.py b/redash/query_runner/influx_db.py
@@ -55,7 +55,15 @@ class InfluxDB(BaseQueryRunner):
     def configuration_schema(cls):
         return {
             "type": "object",
-            "properties": {"url": {"type": "string"}},
+            "properties": {
+                "url": {"type": "string"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
+            },
             "required": ["url"],
         }
 

diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py
@@ -50,6 +50,12 @@ def configuration_schema(cls):
                 "port": {"type": "number"},
                 "user": {"type": "string"},
                 "password": {"type": "string"},
+                "toggle_table_string": {
+                    "type": "string",
+                    "title": "Toggle Table String",
+                    "default": "_v",
+                    "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight.",
+                },
             },
             "required": ["host", "port"],
             "secret": ["password"],