Add sql mode for ClickHouse

CHANGELOG.md

@@ -18,6 +18,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#116](https://github.com/kobsio/kobs/pull/116): Open details when clicking on Jaeger chart nodes.
 - [#119](https://github.com/kobsio/kobs/pull/119): Add Flux plugin to view and reconcile [Flux](https://fluxcd.io) resources.
 - [#122](https://github.com/kobsio/kobs/pull/122): Add ClickHouse plugin, to query show logs ingested by the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin.
+- [#124](https://github.com/kobsio/kobs/pull/124): Add `sql` mode for ClickHouse to execute raw SQL queries.
 
 ### Fixed
 

docs/configuration/plugins.md

@@ -53,7 +53,7 @@ plugins:
 | address | string | Address of the ClickHouse instance. | Yes |
 | username | string | Username to access a ClickHouse instance. | No |
 | password | string | Password to access a ClickHouse instance. | No |
-| type | string | The type which should be used for the ClickHouse instance. Currently the only supported value is `logs`. The `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. |
+| type | string | The type which should be used for the ClickHouse instance. This must be `sql` or `logs`. While the `sql` mode allows you to use raw SQL queries, the `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. |
 
 ## Elasticsearch
 

docs/plugins/clickhouse.md

@@ -15,7 +15,7 @@ The following options can be used for a panel with the ClickHouse plugin:
 
 | Field | Type | Description | Required |
 | ----- | ---- | ----------- | -------- |
-| type | string | Set the type for which you want to use the ClickHouse instance. Currently the only supported value is `logs`. | Yes |
+| type | string | Set the type for which you want to use the ClickHouse instance. This must be `sql` or `logs` | Yes |
 | queries | [[]Query](#query) | A list of queries, which can be selected by the user. | Yes |
 
 ### Query
@@ -24,7 +24,7 @@ The following options can be used for a panel with the ClickHouse plugin:
 | ----- | ---- | ----------- | -------- |
 | name | string | A name for the ClickHouse query, which is displayed in the select box. | Yes |
 | query | string | The query which should be run against ClickHouse. See [Query Syntax](#query-syntax) for more information on the syntax, when ClickHouse is used in the `logs` mode. | Yes |
-| fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. | No |
+| fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. This field is only available for the `logs`. | No |
 
 ```yaml
 ---
@@ -81,7 +81,9 @@ kobs supports multiple operators which can be used in a query to retrieve logs f
 | `<=` | The value of the field must be lower than or equal to the specified value. | `content.response_code<=499` |
 | `~` | The value of the field must match the regular expression. | `content.upstream_cluster~'inbound.*'` |
 
-### Standard Fields
+### Default Fields
+
+In the following you can find a list of fields which are available for each log line. Consider to filter you logs by these fields, to keep your queries fast:
 
 - `timestamp`: The timestamp for when the log line was written.
 - `cluster`: The name of the cluster as it is set by Fluent Bit.

plugins/clickhouse/clickhouse.go

@@ -47,6 +47,35 @@ func (router *Router) getInstance(name string) *instance.Instance {
 	return nil
 }
 
+func (router *Router) getSQL(w http.ResponseWriter, r *http.Request) {
+	name := chi.URLParam(r, "name")
+	query := r.URL.Query().Get("query")
+
+	log.WithFields(logrus.Fields{"name": name, "query": query}).Tracef("getSQL")
+
+	i := router.getInstance(name)
+	if i == nil {
+		errresponse.Render(w, r, nil, http.StatusBadRequest, "Could not find instance name")
+		return
+	}
+
+	rows, columns, err := i.GetSQL(r.Context(), query)
+	if err != nil {
+		errresponse.Render(w, r, err, http.StatusBadRequest, "Could not get result for SQL query")
+		return
+	}
+
+	data := struct {
+		Rows    [][]interface{} `json:"rows"`
+		Columns []string        `json:"columns"`
+	}{
+		rows,
+		columns,
+	}
+
+	render.JSON(w, r, data)
+}
+
 // getLogs implements the special handling when the user selected the "logs" options for the "view" configuration. This
 // options is intended to use together with the kobsio/fluent-bit-clickhouse Fluent Bit plugin and provides a custom
 // query language to get the logs from ClickHouse.
@@ -101,11 +130,17 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	render.JSON(w, r, logsResponse{
-		Documents: documents,
-		Fields:    fields,
-		Offset:    newOffset,
-	})
+	data := struct {
+		Documents []map[string]interface{} `json:"documents"`
+		Fields    []string                 `json:"fields"`
+		Offset    int64                    `json:"offset"`
+	}{
+		documents,
+		fields,
+		newOffset,
+	}
+
+	render.JSON(w, r, data)
 }
 
 // Register returns a new router which can be used in the router for the kobs rest api.
@@ -139,6 +174,7 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi
 		instances,
 	}
 
+	router.Get("/sql/{name}", router.getSQL)
 	router.Get("/logs/{name}", router.getLogs)
 
 	return router

plugins/clickhouse/pkg/instance/instance.go

@@ -36,6 +36,41 @@ type Instance struct {
 	client   *sql.DB
 }
 
+// GetSQL returns all rows for the user provided SQL query.
+func (i *Instance) GetSQL(ctx context.Context, query string) ([][]interface{}, []string, error) {
+	rows, err := i.client.QueryContext(ctx, query)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+
+	var columns []string
+	columns, err = rows.Columns()
+	if err != nil {
+		return nil, nil, err
+	}
+	columnsLen := len(columns)
+
+	var result [][]interface{}
+
+	for rows.Next() {
+		var r []interface{}
+		r = make([]interface{}, columnsLen)
+
+		for i := 0; i < columnsLen; i++ {
+			r[i] = new(interface{})
+		}
+
+		if err := rows.Scan(r...); err != nil {
+			return nil, nil, err
+		}
+
+		result = append(result, r)
+	}
+
+	return result, columns, nil
+}
+
 // GetLogs parses the given query into the sql syntax, which is then run against the ClickHouse instance. The returned
 // rows are converted into a document schema which can be used by our UI.
 func (i *Instance) GetLogs(ctx context.Context, query string, limit, offset, timeStart, timeEnd int64) ([]map[string]interface{}, []string, int64, error) {

plugins/clickhouse/src/components/page/LogsToolbar.tsx

@@ -70,7 +70,7 @@ const LogsToolbar: React.FunctionComponent<ILogsToolbarProps> = ({
   };
 
   return (
-    <Toolbar id="elasticsearch-toolbar" style={{ paddingBottom: '0px', zIndex: 300 }}>
+    <Toolbar id="clickhouse-logs-toolbar" style={{ paddingBottom: '0px', zIndex: 300 }}>
       <ToolbarContent style={{ padding: '0px' }}>
         <ToolbarToggleGroup style={{ width: '100%' }} toggleIcon={<FilterIcon />} breakpoint="lg">
           <ToolbarGroup style={{ alignItems: 'flex-start', width: '100%' }}>

plugins/clickhouse/src/components/page/Page.tsx

@@ -2,6 +2,7 @@ import React from 'react';
 
 import { IPluginPageProps } from '@kobsio/plugin-core';
 import LogsPage from './LogsPage';
+import SQLPage from './SQLPage';
 
 const Page: React.FunctionComponent<IPluginPageProps> = ({
   name,
@@ -11,6 +12,8 @@ const Page: React.FunctionComponent<IPluginPageProps> = ({
 }: IPluginPageProps) => {
   if (options && options.type && options.type === 'logs') {
     return <LogsPage name={name} displayName={displayName} description={description} />;
+  } else if (options && options.type && options.type === 'sql') {
+    return <SQLPage name={name} displayName={displayName} description={description} />;
   }
 
   return null;

plugins/clickhouse/src/components/page/SQL.tsx

@@ -0,0 +1,79 @@
+import { Alert, AlertActionLink, AlertVariant, Card, Spinner } from '@patternfly/react-core';
+import { QueryObserverResult, useQuery } from 'react-query';
+import React from 'react';
+import { useHistory } from 'react-router-dom';
+
+import { ISQLData } from '../../utils/interfaces';
+import SQLTable from '../panel/SQLTable';
+
+interface ISQLProps {
+  name: string;
+  query: string;
+}
+
+const SQL: React.FunctionComponent<ISQLProps> = ({ name, query }: ISQLProps) => {
+  const history = useHistory();
+
+  const { isError, isFetching, error, data, refetch } = useQuery<ISQLData, Error>(
+    ['clickhouse/sql', query],
+    async () => {
+      try {
+        const response = await fetch(`/api/plugins/clickhouse/sql/${name}?query=${query}`, {
+          method: 'get',
+        });
+        const json = await response.json();
+
+        if (response.status >= 200 && response.status < 300) {
+          return json;
+        } else {
+          if (json.error) {
+            throw new Error(json.error);
+          } else {
+            throw new Error('An unknown error occured');
+          }
+        }
+      } catch (err) {
+        throw err;
+      }
+    },
+  );
+
+  if (isFetching) {
+    return (
+      <div className="pf-u-text-align-center">
+        <Spinner />
+      </div>
+    );
+  }
+
+  if (isError) {
+    return (
+      <Alert
+        variant={AlertVariant.danger}
+        title="Could not get result for SQL query"
+        actionLinks={
+          <React.Fragment>
+            <AlertActionLink onClick={(): void => history.push('/')}>Home</AlertActionLink>
+            <AlertActionLink onClick={(): Promise<QueryObserverResult<ISQLData, Error>> => refetch()}>
+              Retry
+            </AlertActionLink>
+          </React.Fragment>
+        }
+      >
+        <p>{error?.message}</p>
+      </Alert>
+    );
+  }
+
+  if (!data) {
+    return null;
+  }
+
+  return (
+    <Card isCompact={true} style={{ maxWidth: '100%', overflowX: 'scroll' }}>
+      <SQLTable columns={data.columns} rows={data.rows} />
+    </Card>
+  );
+};
+
+export default SQL;

plugins/clickhouse/src/components/page/SQLPage.tsx

@@ -0,0 +1,47 @@
+import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core';
+import React, { useEffect, useState } from 'react';
+import { useHistory, useLocation } from 'react-router-dom';
+
+import { IPluginPageProps } from '@kobsio/plugin-core';
+import SQL from './SQL';
+import SQLToolbar from './SQLToolbar';
+import { getQueryFromSearch } from '../../utils/helpers';
+
+const SQLPage: React.FunctionComponent<IPluginPageProps> = ({ name, displayName, description }: IPluginPageProps) => {
+  const location = useLocation();
+  const history = useHistory();
+  const [query, setQuery] = useState<string>(getQueryFromSearch(location.search));
+
+  // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options
+  // state we change the URL parameters.
+  const changeOptions = (q: string): void => {
+    history.push({
+      pathname: location.pathname,
+      search: `?query=${q}`,
+    });
+  };
+
+  // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed
+  // via the changeOptions function. When the search location is changed we modify the options state.
+  useEffect(() => {
+    setQuery(getQueryFromSearch(location.search));
+  }, [location.search]);
+
+  return (
+    <React.Fragment>
+      <PageSection variant={PageSectionVariants.light}>
+        <Title headingLevel="h6" size="xl">
+          {displayName}
+        </Title>
+        <p>{description}</p>
+        <SQLToolbar query={query} setQuery={changeOptions} />
+      </PageSection>
+
+      <PageSection style={{ minHeight: '100%' }} variant={PageSectionVariants.default}>
+        {query.length > 0 && <SQL name={name} query={query} />}
+      </PageSection>
+    </React.Fragment>
+  );
+};
+
+export default SQLPage;

plugins/clickhouse/src/components/page/SQLToolbar.tsx

@@ -0,0 +1,65 @@
+import {
+  Button,
+  ButtonVariant,
+  TextArea,
+  Toolbar,
+  ToolbarContent,
+  ToolbarGroup,
+  ToolbarItem,
+  ToolbarToggleGroup,
+} from '@patternfly/react-core';
+import { FilterIcon, SearchIcon } from '@patternfly/react-icons';
+import React, { useState } from 'react';
+
+interface ISQLToolbarProps {
+  query: string;
+  setQuery: (data: string) => void;
+}
+
+const SQLToolbar: React.FunctionComponent<ISQLToolbarProps> = ({ query, setQuery }: ISQLToolbarProps) => {
+  const [data, setData] = useState<string>(query);
+
+  // changeQuery changes the value of a query.
+  const changeQuery = (value: string): void => {
+    setData(value);
+  };
+
+  // onEnter is used to detect if the user pressed the "ENTER" key. If this is the case we are calling the setOptions
+  // function to trigger the search.
+  // use "SHIFT" + "ENTER".
+  const onEnter = (e: React.KeyboardEvent<HTMLTextAreaElement> | undefined): void => {
+    if (e?.key === 'Enter' && !e.shiftKey) {
+      e.preventDefault();
+      setQuery(data);
+    }
+  };
+
+  return (
+    <Toolbar id="clickhouse-sql-toolbar" style={{ paddingBottom: '0px', zIndex: 300 }}>
+      <ToolbarContent style={{ padding: '0px' }}>
+        <ToolbarToggleGroup style={{ width: '100%' }} toggleIcon={<FilterIcon />} breakpoint="lg">
+          <ToolbarGroup style={{ alignItems: 'flex-start', width: '100%' }}>
+            <ToolbarItem style={{ width: '100%' }}>
+              <TextArea
+                aria-label="Query"
+                resizeOrientation="vertical"
+                rows={1}
+                type="text"
+                value={data}
+                onChange={changeQuery}
+                onKeyDown={onEnter}
+              />
+            </ToolbarItem>
+            <ToolbarItem>
+              <Button variant={ButtonVariant.primary} icon={<SearchIcon />} onClick={(): void => setQuery(data)}>
+                Search
+              </Button>
+            </ToolbarItem>
+          </ToolbarGroup>
+        </ToolbarToggleGroup>
+      </ToolbarContent>
+    </Toolbar>
+  );
+};
+
+export default SQLToolbar;