From 89e6d8394703940c2c690498f5964411f09785e1 Mon Sep 17 00:00:00 2001 From: ricoberger Date: Sun, 29 Aug 2021 17:26:12 +0200 Subject: [PATCH] Add sql mode for ClickHouse This adds a new mode called "sql" for the ClickHouse plugin, which allows a user to execute raw SQL queries against the configured ClickHouse instance. The result of the queries will be shown in a table. --- CHANGELOG.md | 1 + docs/configuration/plugins.md | 2 +- docs/plugins/clickhouse.md | 8 +- plugins/clickhouse/clickhouse.go | 46 ++++++- plugins/clickhouse/pkg/instance/instance.go | 35 ++++++ .../src/components/page/LogsToolbar.tsx | 2 +- .../clickhouse/src/components/page/Page.tsx | 3 + .../clickhouse/src/components/page/SQL.tsx | 79 ++++++++++++ .../src/components/page/SQLPage.tsx | 47 ++++++++ .../src/components/page/SQLToolbar.tsx | 65 ++++++++++ .../clickhouse/src/components/panel/Panel.tsx | 9 +- .../clickhouse/src/components/panel/SQL.tsx | 114 ++++++++++++++++++ .../src/components/panel/SQLActions.tsx | 30 +++++ .../src/components/panel/SQLTable.tsx | 37 ++++++ plugins/clickhouse/src/utils/helpers.ts | 9 +- plugins/clickhouse/src/utils/interfaces.ts | 6 + 16 files changed, 481 insertions(+), 12 deletions(-) create mode 100644 plugins/clickhouse/src/components/page/SQL.tsx create mode 100644 plugins/clickhouse/src/components/page/SQLPage.tsx create mode 100644 plugins/clickhouse/src/components/page/SQLToolbar.tsx create mode 100644 plugins/clickhouse/src/components/panel/SQL.tsx create mode 100644 plugins/clickhouse/src/components/panel/SQLActions.tsx create mode 100644 plugins/clickhouse/src/components/panel/SQLTable.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 31bac0bef..5e227bc4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan - [#116](https://github.com/kobsio/kobs/pull/116): Open details when clicking on Jaeger chart nodes. - [#119](https://github.com/kobsio/kobs/pull/119): Add Flux plugin to view and reconcile [Flux](https://fluxcd.io) resources. - [#122](https://github.com/kobsio/kobs/pull/122): Add ClickHouse plugin, to query show logs ingested by the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) Fluent Bit plugin. +- [#124](https://github.com/kobsio/kobs/pull/124): Add `sql` mode for ClickHouse to execute raw SQL queries. ### Fixed diff --git a/docs/configuration/plugins.md b/docs/configuration/plugins.md index 5ad586065..72fdd3d47 100644 --- a/docs/configuration/plugins.md +++ b/docs/configuration/plugins.md @@ -53,7 +53,7 @@ plugins: | address | string | Address of the ClickHouse instance. | Yes | | username | string | Username to access a ClickHouse instance. | No | | password | string | Password to access a ClickHouse instance. | No | -| type | string | The type which should be used for the ClickHouse instance. Currently the only supported value is `logs`. The `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. | +| type | string | The type which should be used for the ClickHouse instance. This must be `sql` or `logs`. While the `sql` mode allows you to use raw SQL queries, the `logs` mode should be used together with the [kobsio/fluent-bit-clickhouse](https://github.com/kobsio/fluent-bit-clickhouse) plugin to collect logs via Fluent Bit and save them in ClickHouse. | ## Elasticsearch diff --git a/docs/plugins/clickhouse.md b/docs/plugins/clickhouse.md index 97f444838..27fb879b3 100644 --- a/docs/plugins/clickhouse.md +++ b/docs/plugins/clickhouse.md @@ -15,7 +15,7 @@ The following options can be used for a panel with the ClickHouse plugin: | Field | Type | Description | Required | | ----- | ---- | ----------- | -------- | -| type | string | Set the type for which you want to use the ClickHouse instance. Currently the only supported value is `logs`. | Yes | +| type | string | Set the type for which you want to use the ClickHouse instance. This must be `sql` or `logs` | Yes | | queries | [[]Query](#query) | A list of queries, which can be selected by the user. | Yes | ### Query @@ -24,7 +24,7 @@ The following options can be used for a panel with the ClickHouse plugin: | ----- | ---- | ----------- | -------- | | name | string | A name for the ClickHouse query, which is displayed in the select box. | Yes | | query | string | The query which should be run against ClickHouse. See [Query Syntax](#query-syntax) for more information on the syntax, when ClickHouse is used in the `logs` mode. | Yes | -| fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. | No | +| fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. This field is only available for the `logs`. | No | ```yaml --- @@ -81,7 +81,9 @@ kobs supports multiple operators which can be used in a query to retrieve logs f | `<=` | The value of the field must be lower than or equal to the specified value. | `content.response_code<=499` | | `~` | The value of the field must match the regular expression. | `content.upstream_cluster~'inbound.*'` | -### Standard Fields +### Default Fields + +In the following you can find a list of fields which are available for each log line. Consider to filter you logs by these fields, to keep your queries fast: - `timestamp`: The timestamp for when the log line was written. - `cluster`: The name of the cluster as it is set by Fluent Bit. diff --git a/plugins/clickhouse/clickhouse.go b/plugins/clickhouse/clickhouse.go index fac623abb..99c174230 100644 --- a/plugins/clickhouse/clickhouse.go +++ b/plugins/clickhouse/clickhouse.go @@ -47,6 +47,35 @@ func (router *Router) getInstance(name string) *instance.Instance { return nil } +func (router *Router) getSQL(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + query := r.URL.Query().Get("query") + + log.WithFields(logrus.Fields{"name": name, "query": query}).Tracef("getSQL") + + i := router.getInstance(name) + if i == nil { + errresponse.Render(w, r, nil, http.StatusBadRequest, "Could not find instance name") + return + } + + rows, columns, err := i.GetSQL(r.Context(), query) + if err != nil { + errresponse.Render(w, r, err, http.StatusBadRequest, "Could not get result for SQL query") + return + } + + data := struct { + Rows [][]interface{} `json:"rows"` + Columns []string `json:"columns"` + }{ + rows, + columns, + } + + render.JSON(w, r, data) +} + // getLogs implements the special handling when the user selected the "logs" options for the "view" configuration. This // options is intended to use together with the kobsio/fluent-bit-clickhouse Fluent Bit plugin and provides a custom // query language to get the logs from ClickHouse. @@ -101,11 +130,17 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) { return } - render.JSON(w, r, logsResponse{ - Documents: documents, - Fields: fields, - Offset: newOffset, - }) + data := struct { + Documents []map[string]interface{} `json:"documents"` + Fields []string `json:"fields"` + Offset int64 `json:"offset"` + }{ + documents, + fields, + newOffset, + } + + render.JSON(w, r, data) } // Register returns a new router which can be used in the router for the kobs rest api. @@ -139,6 +174,7 @@ func Register(clusters *clusters.Clusters, plugins *plugin.Plugins, config Confi instances, } + router.Get("/sql/{name}", router.getSQL) router.Get("/logs/{name}", router.getLogs) return router diff --git a/plugins/clickhouse/pkg/instance/instance.go b/plugins/clickhouse/pkg/instance/instance.go index 9601f0b67..53779763d 100644 --- a/plugins/clickhouse/pkg/instance/instance.go +++ b/plugins/clickhouse/pkg/instance/instance.go @@ -36,6 +36,41 @@ type Instance struct { client *sql.DB } +// GetSQL returns all rows for the user provided SQL query. +func (i *Instance) GetSQL(ctx context.Context, query string) ([][]interface{}, []string, error) { + rows, err := i.client.QueryContext(ctx, query) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + var columns []string + columns, err = rows.Columns() + if err != nil { + return nil, nil, err + } + columnsLen := len(columns) + + var result [][]interface{} + + for rows.Next() { + var r []interface{} + r = make([]interface{}, columnsLen) + + for i := 0; i < columnsLen; i++ { + r[i] = new(interface{}) + } + + if err := rows.Scan(r...); err != nil { + return nil, nil, err + } + + result = append(result, r) + } + + return result, columns, nil +} + // GetLogs parses the given query into the sql syntax, which is then run against the ClickHouse instance. The returned // rows are converted into a document schema which can be used by our UI. func (i *Instance) GetLogs(ctx context.Context, query string, limit, offset, timeStart, timeEnd int64) ([]map[string]interface{}, []string, int64, error) { diff --git a/plugins/clickhouse/src/components/page/LogsToolbar.tsx b/plugins/clickhouse/src/components/page/LogsToolbar.tsx index 2cf4bcac4..4f7b25ff9 100644 --- a/plugins/clickhouse/src/components/page/LogsToolbar.tsx +++ b/plugins/clickhouse/src/components/page/LogsToolbar.tsx @@ -70,7 +70,7 @@ const LogsToolbar: React.FunctionComponent = ({ }; return ( - + } breakpoint="lg"> diff --git a/plugins/clickhouse/src/components/page/Page.tsx b/plugins/clickhouse/src/components/page/Page.tsx index bf26fb33e..6aa0f4799 100644 --- a/plugins/clickhouse/src/components/page/Page.tsx +++ b/plugins/clickhouse/src/components/page/Page.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { IPluginPageProps } from '@kobsio/plugin-core'; import LogsPage from './LogsPage'; +import SQLPage from './SQLPage'; const Page: React.FunctionComponent = ({ name, @@ -11,6 +12,8 @@ const Page: React.FunctionComponent = ({ }: IPluginPageProps) => { if (options && options.type && options.type === 'logs') { return ; + } else if (options && options.type && options.type === 'sql') { + return ; } return null; diff --git a/plugins/clickhouse/src/components/page/SQL.tsx b/plugins/clickhouse/src/components/page/SQL.tsx new file mode 100644 index 000000000..95149962a --- /dev/null +++ b/plugins/clickhouse/src/components/page/SQL.tsx @@ -0,0 +1,79 @@ +import { Alert, AlertActionLink, AlertVariant, Card, Spinner } from '@patternfly/react-core'; +import { QueryObserverResult, useQuery } from 'react-query'; +import React from 'react'; +import { useHistory } from 'react-router-dom'; + +import { ISQLData } from '../../utils/interfaces'; +import SQLTable from '../panel/SQLTable'; + +interface ISQLProps { + name: string; + query: string; +} + +const SQL: React.FunctionComponent = ({ name, query }: ISQLProps) => { + const history = useHistory(); + + const { isError, isFetching, error, data, refetch } = useQuery( + ['clickhouse/sql', query], + async () => { + try { + const response = await fetch(`/api/plugins/clickhouse/sql/${name}?query=${query}`, { + method: 'get', + }); + const json = await response.json(); + + if (response.status >= 200 && response.status < 300) { + return json; + } else { + if (json.error) { + throw new Error(json.error); + } else { + throw new Error('An unknown error occured'); + } + } + } catch (err) { + throw err; + } + }, + ); + + if (isFetching) { + return ( +
+ +
+ ); + } + + if (isError) { + return ( + + history.push('/')}>Home + > => refetch()}> + Retry + + + } + > +

{error?.message}

+ + ); + } + + if (!data) { + return null; + } + + return ( + + + + ); +}; + +export default SQL; diff --git a/plugins/clickhouse/src/components/page/SQLPage.tsx b/plugins/clickhouse/src/components/page/SQLPage.tsx new file mode 100644 index 000000000..84136e8aa --- /dev/null +++ b/plugins/clickhouse/src/components/page/SQLPage.tsx @@ -0,0 +1,47 @@ +import { PageSection, PageSectionVariants, Title } from '@patternfly/react-core'; +import React, { useEffect, useState } from 'react'; +import { useHistory, useLocation } from 'react-router-dom'; + +import { IPluginPageProps } from '@kobsio/plugin-core'; +import SQL from './SQL'; +import SQLToolbar from './SQLToolbar'; +import { getQueryFromSearch } from '../../utils/helpers'; + +const SQLPage: React.FunctionComponent = ({ name, displayName, description }: IPluginPageProps) => { + const location = useLocation(); + const history = useHistory(); + const [query, setQuery] = useState(getQueryFromSearch(location.search)); + + // changeOptions is used to change the options for an ClickHouse query. Instead of directly modifying the options + // state we change the URL parameters. + const changeOptions = (q: string): void => { + history.push({ + pathname: location.pathname, + search: `?query=${q}`, + }); + }; + + // useEffect is used to set the options every time the search location for the current URL changes. The URL is changed + // via the changeOptions function. When the search location is changed we modify the options state. + useEffect(() => { + setQuery(getQueryFromSearch(location.search)); + }, [location.search]); + + return ( + + + + {displayName} + +

{description}

+ + + + + {query.length > 0 && } + + + ); +}; + +export default SQLPage; diff --git a/plugins/clickhouse/src/components/page/SQLToolbar.tsx b/plugins/clickhouse/src/components/page/SQLToolbar.tsx new file mode 100644 index 000000000..8506a6522 --- /dev/null +++ b/plugins/clickhouse/src/components/page/SQLToolbar.tsx @@ -0,0 +1,65 @@ +import { + Button, + ButtonVariant, + TextArea, + Toolbar, + ToolbarContent, + ToolbarGroup, + ToolbarItem, + ToolbarToggleGroup, +} from '@patternfly/react-core'; +import { FilterIcon, SearchIcon } from '@patternfly/react-icons'; +import React, { useState } from 'react'; + +interface ISQLToolbarProps { + query: string; + setQuery: (data: string) => void; +} + +const SQLToolbar: React.FunctionComponent = ({ query, setQuery }: ISQLToolbarProps) => { + const [data, setData] = useState(query); + + // changeQuery changes the value of a query. + const changeQuery = (value: string): void => { + setData(value); + }; + + // onEnter is used to detect if the user pressed the "ENTER" key. If this is the case we are calling the setOptions + // function to trigger the search. + // use "SHIFT" + "ENTER". + const onEnter = (e: React.KeyboardEvent | undefined): void => { + if (e?.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + setQuery(data); + } + }; + + return ( + + + } breakpoint="lg"> + + +