[SQL] initial commit for SQL language selector

Only adds the quick startup for OpenSearch cluster with a SQL plugin and observability with: ``` yarn opensearch snapshot --sql ``` Also, adds SQL to the language selector stolen shamelessly from opensearch-project#5623 Next steps to intercept and send to SQL API or just transform basic syntax to DSL Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
kavilla · Mar 12, 2024 · 5f6916d · 5f6916d
1 parent 8d3df17
commit 5f6916d
Show file tree

Hide file tree

Showing 12 changed files with 150 additions and 101 deletions.
diff --git a/packages/osd-opensearch/src/cli_commands/snapshot.js b/packages/osd-opensearch/src/cli_commands/snapshot.js
@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js
@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js
@@ -249,6 +249,7 @@ export default function (program) {
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
       .option('--security', 'Run the dev server using security defaults')
+      .option('--sql', 'Run the dev server using SQL/PPL defaults')
       .option('--dist', 'Use production assets from osd/optimizer')
       .option(
         '--no-base-path',

diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts
@@ -35,6 +35,7 @@ import { buildQueryFromLucene } from './from_lucene';
 import { IIndexPattern } from '../../index_patterns';
 import { Filter } from '../filters';
 import { Query } from '../../query/types';
+import { buildQueryFromSql } from './from_sql';
 
 export interface OpenSearchQueryConfig {
   allowLeadingWildcards: boolean;
@@ -64,7 +65,17 @@ export function buildOpenSearchQuery(
   queries = Array.isArray(queries) ? queries : [queries];
   filters = Array.isArray(filters) ? filters : [filters];
 
-  const validQueries = queries.filter((query) => has(query, 'query'));
+  // TODO: SQL make this combinable. SQL needs to support DSL
+  // console.log('queries', queries);
+  const sqlQueries = queries.filter((query) => query.language === 'SQL');
+  if (sqlQueries.length > 0) {
+    // console.log('sqlQueries', sqlQueries);
+    return buildQueryFromSql(sqlQueries, config.dateFormatTZ);
+  }
+
+  const validQueries = queries
+    .filter((query) => query.language !== 'SQL')
+    .filter((query) => has(query, 'query'));
   const queriesByLanguage = groupBy(validQueries, 'language');
   const kueryQuery = buildQueryFromKuery(
     indexPattern,

diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/from_sql.ts b/src/plugins/data/common/opensearch_query/opensearch_query/from_sql.ts
@@ -0,0 +1,21 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { decorateQuery } from './decorate_query';
+import { getIndexPatternFromSql, sqlStringToDsl } from './sql_string_to_dsl';
+import { Query } from '../../query/types';
+
+export function buildQueryFromSql(queries: Query[], dateFormatTZ?: string) {
+  const combinedQueries = (queries || []).map((query) => {
+    const indexPattern = getIndexPatternFromSql(query.query);
+    const queryDsl = sqlStringToDsl(query.query);
+
+    return decorateQuery(queryDsl, indexPattern, dateFormatTZ);
+  });
+
+  return {
+    combinedQueries,
+  };
+}
diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/index.ts b/src/plugins/data/common/opensearch_query/opensearch_query/index.ts
@@ -31,5 +31,6 @@
 export { buildOpenSearchQuery, OpenSearchQueryConfig } from './build_opensearch_query';
 export { buildQueryFromFilters } from './from_filters';
 export { luceneStringToDsl } from './lucene_string_to_dsl';
+export { getIndexPatternFromSql, sqlStringToDsl } from './sql_string_to_dsl';
 export { decorateQuery } from './decorate_query';
 export { getOpenSearchQueryConfig } from './get_opensearch_query_config';
diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/sql_string_to_dsl.ts b/src/plugins/data/common/opensearch_query/opensearch_query/sql_string_to_dsl.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { isString } from 'lodash';
+import { DslQuery } from './opensearch_query_dsl';
+
+export function getIndexPatternFromSql(query: string | any) {
+  const from = query.match(new RegExp(/FROM\s+([\w*-.!@$^()~;]+)/, 'i'));
+  if (from) {
+    return from[1];
+  }
+  return '';
+}
+
+export function sqlStringToDsl(query: string | any): DslQuery {
+  if (isString(query)) {
+    return { query_string: { query } };
+  }
+
+  return query;
+}
diff --git a/src/plugins/data/public/search/expressions/opensearchdsl.ts b/src/plugins/data/public/search/expressions/opensearchdsl.ts
@@ -28,6 +28,8 @@
  * under the License.
  */
 
+// TODO: SQL this file seems important
+
 import { i18n } from '@osd/i18n';
 import {
   OpenSearchDashboardsContext,

diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts
@@ -28,6 +28,8 @@
  * under the License.
  */
 
+// TODO: SQL this file seems important
+
 import { get, trimEnd, debounce } from 'lodash';
 import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs';
 import { catchError, finalize } from 'rxjs/operators';
@@ -236,6 +238,11 @@ export class SearchInterceptor {
       });
       this.pendingCount$.next(this.pendingCount$.getValue() + 1);
 
+      // TODO: SQL this isn't the right place but if core includes SQL then we dont need to do this
+      // TODO: hack setting to undefined
+      // console.log(request);
+      // console.log(options);
+
       return this.runSearch(
         request,
         combinedSignal,

diff --git a/src/plugins/data/public/ui/query_string_input/_index.scss b/src/plugins/data/public/ui/query_string_input/_index.scss
@@ -1 +1,2 @@
 @import "./query_bar";
+@import "./language_switcher"
diff --git a/src/plugins/data/public/ui/query_string_input/_language_switcher.scss b/src/plugins/data/public/ui/query_string_input/_language_switcher.scss
@@ -0,0 +1,4 @@
+// From: https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5623
+.languageSwitcher {
+  max-width: 150px;
+}
diff --git a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx
@@ -28,21 +28,9 @@
  * under the License.
  */
 
-import {
-  EuiButtonEmpty,
-  EuiForm,
-  EuiFormRow,
-  EuiLink,
-  EuiPopover,
-  EuiPopoverTitle,
-  EuiSpacer,
-  EuiSwitch,
-  EuiText,
-  PopoverAnchorPosition,
-} from '@elastic/eui';
-import { FormattedMessage } from '@osd/i18n/react';
-import React, { useState } from 'react';
-import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public';
+import { EuiComboBox, EuiComboBoxOptionOption, PopoverAnchorPosition } from '@elastic/eui';
+import { i18n } from '@osd/i18n';
+import React from 'react';
 
 interface Props {
   language: string;
@@ -51,93 +39,53 @@ interface Props {
 }
 
 export function QueryLanguageSwitcher(props: Props) {
-  const osdDQLDocs = useOpenSearchDashboards().services.docLinks?.links.opensearchDashboards.dql
-    .base;
-  const [isPopoverOpen, setIsPopoverOpen] = useState(false);
-  const luceneLabel = (
-    <FormattedMessage id="data.query.queryBar.luceneLanguageName" defaultMessage="Lucene" />
-  );
-  const dqlLabel = (
-    <FormattedMessage id="data.query.queryBar.dqlLanguageName" defaultMessage="DQL" />
-  );
-  const dqlFullName = (
-    <FormattedMessage
-      id="data.query.queryBar.dqlFullLanguageName"
-      defaultMessage="OpenSearch Dashboards Query Language"
-    />
-  );
+  const luceneLabel = i18n.translate('data.query.queryBar.luceneLanguageName', {
+    defaultMessage: 'Lucene',
+  });
+  const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', {
+    defaultMessage: 'DQL',
+  });
+  const sqlLabel = i18n.translate('data.query.queryBar.sqlLanguageName', {
+    defaultMessage: 'SQL',
+  });
 
-  const button = (
-    <EuiButtonEmpty
-      size="xs"
-      onClick={() => setIsPopoverOpen(!isPopoverOpen)}
-      className="euiFormControlLayout__append dqlQueryBar__languageSwitcherButton"
-      data-test-subj={'switchQueryLanguageButton'}
-    >
-      {props.language === 'lucene' ? luceneLabel : dqlLabel}
-    </EuiButtonEmpty>
-  );
+  const languageOptions: EuiComboBoxOptionOption[] = [
+    {
+      label: luceneLabel,
+      value: 'lucene',
+    },
+    {
+      label: dqlLabel,
+      value: 'dql',
+    },
+    {
+      label: sqlLabel,
+      value: 'sql',
+      // TODO: add option to disable if SQL is not supported
+      // disabled: true,
+    },
+  ];
 
-  return (
-    <EuiPopover
-      id="queryLanguageSwitcherPopover"
-      anchorClassName="euiFormControlLayout__append"
-      ownFocus
-      anchorPosition={props.anchorPosition || 'downRight'}
-      button={button}
-      isOpen={isPopoverOpen}
-      closePopover={() => setIsPopoverOpen(false)}
-      repositionOnScroll
-    >
-      <EuiPopoverTitle>
-        <FormattedMessage
-          id="data.query.queryBar.syntaxOptionsTitle"
-          defaultMessage="Syntax options"
-        />
-      </EuiPopoverTitle>
-      <div style={{ width: '350px' }}>
-        <EuiText>
-          <p>
-            <FormattedMessage
-              id="data.query.queryBar.syntaxOptionsDescription"
-              defaultMessage="The {docsLink} (DQL) offers a simplified query
-              syntax and support for scripted fields. If you turn off DQL,
-              OpenSearch Dashboards uses Lucene."
-              values={{
-                docsLink: (
-                  <EuiLink href={osdDQLDocs} target="_blank">
-                    {dqlFullName}
-                  </EuiLink>
-                ),
-              }}
-            />
-          </p>
-        </EuiText>
+  const selectedLanguage = {
+    label: props.language === 'kuery' ? 'DQL' : props.language,
+  };
 
-        <EuiSpacer size="m" />
+  const handleLanguageChange = (newLanguage: EuiComboBoxOptionOption[]) => {
+    const queryLanguage = newLanguage[0].label === 'DQL' ? 'kuery' : newLanguage[0].label;
+    props.onSelectLanguage(queryLanguage);
+  };
 
-        <EuiForm>
-          <EuiFormRow label={dqlFullName}>
-            <EuiSwitch
-              id="queryEnhancementOptIn"
-              name="popswitch"
-              label={
-                props.language === 'kuery' ? (
-                  <FormattedMessage id="data.query.queryBar.dqlOnLabel" defaultMessage="On" />
-                ) : (
-                  <FormattedMessage id="data.query.queryBar.dqlOffLabel" defaultMessage="Off" />
-                )
-              }
-              checked={props.language === 'kuery'}
-              onChange={() => {
-                const newLanguage = props.language === 'lucene' ? 'kuery' : 'lucene';
-                props.onSelectLanguage(newLanguage);
-              }}
-              data-test-subj="languageToggle"
-            />
-          </EuiFormRow>
-        </EuiForm>
-      </div>
-    </EuiPopover>
+  return (
+    <EuiComboBox
+      className="languageSwitcher"
+      data-test-subj="languageSelect"
+      // TODO: FIX THIS ANY
+      options={languageOptions}
+      selectedOptions={[selectedLanguage]}
+      onChange={handleLanguageChange}
+      singleSelection={{ asPlainText: true }}
+      isClearable={false}
+      async
+    />
   );
 }
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		@import "./query_bar";
		@import "./language_switcher"