[MQL] support different query languages

Only adds the quick startup for OpenSearch cluster with a SQL plugin and observability with: ``` yarn opensearch snapshot --sql ``` Also, adds SQL to the language selector stolen shamelessly from opensearch-project#5623 Next steps to intercept and send to SQL API or just transform basic syntax to DSL Signed-off-by: Kawika Avilla <kavilla414@gmail.com> Working query enhance Signed-off-by: Kawika Avilla <kavilla414@gmail.com> Reget search enhance Signed-off-by: Kawika Avilla <kavilla414@gmail.com> data frames Signed-off-by: Kawika Avilla <kavilla414@gmail.com> temp state need to pass df Signed-off-by: Kawika Avilla <kavilla414@gmail.com> sending data frame Signed-off-by: Kawika Avilla <kavilla414@gmail.com> gets df type Signed-off-by: Kawika Avilla <kavilla414@gmail.com> add some small ui things Signed-off-by: Kawika Avilla <kavilla414@gmail.com> move back query language switcher Signed-off-by: Kawika Avilla <kavilla414@gmail.com> updating side panel Signed-off-by: Kawika Avilla <kavilla414@gmail.com> add ui config on query enhancement Signed-off-by: Kawika Avilla <kavilla414@gmail.com> support query string input Signed-off-by: Kawika Avilla <kavilla414@gmail.com> update to say search bar Signed-off-by: Kawika Avilla <kavilla414@gmail.com> update ppl to call once Signed-off-by: Kawika Avilla <kavilla414@gmail.com> update when fields update Signed-off-by: Kawika Avilla <kavilla414@gmail.com> add unknown Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
kavilla · Apr 22, 2024 · 8a6b524 · 8a6b524
1 parent 7bff6d4
commit 8a6b524
Show file tree

Hide file tree

Showing 59 changed files with 1,051 additions and 164 deletions.
diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml
@@ -313,3 +313,5 @@
 
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
+opensearch_alerting.enabled: false
+opensearch_security.enabled: false
diff --git a/package.json b/package.json
@@ -136,7 +136,7 @@
   },
   "dependencies": {
     "@aws-crypto/client-node": "^3.1.1",
-    "@elastic/datemath": "5.0.3",
+    "@elastic/datemath": "link:packages/opensearch-datemath",
     "@elastic/eui": "npm:@opensearch-project/oui@1.5.1",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0",

diff --git a/packages/opensearch-datemath/index.d.ts b/packages/opensearch-datemath/index.d.ts
@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

diff --git a/packages/opensearch-datemath/index.js b/packages/opensearch-datemath/index.js
@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d) => moment.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

diff --git a/packages/osd-opensearch/src/cli_commands/snapshot.js b/packages/osd-opensearch/src/cli_commands/snapshot.js
@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js
@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js
@@ -249,6 +249,7 @@ export default function (program) {
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
       .option('--security', 'Run the dev server using security defaults')
+      .option('--sql', 'Run the dev server using SQL/PPL defaults')
       .option('--dist', 'Use production assets from osd/optimizer')
       .option(
         '--no-base-path',

diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts
@@ -60,4 +60,5 @@ export const UI_SETTINGS = {
   INDEXPATTERN_PLACEHOLDER: 'indexPattern:placeholder',
   FILTERS_PINNED_BY_DEFAULT: 'filters:pinnedByDefault',
   FILTERS_EDITOR_SUGGEST_VALUES: 'filterEditor:suggestValues',
+  DATAFRAME_HYDRATION_STRATEGY: 'dataframe:hydrationStrategy',
 } as const;
diff --git a/src/plugins/data/common/data_frames/_df_cache.ts b/src/plugins/data/common/data_frames/_df_cache.ts
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import { IDataFrame } from '..';
+
+export interface DfCache {
+  get: () => IDataFrame | undefined;
+  set: (value: IDataFrame) => IDataFrame;
+  clear: () => void;
+}
+
+export function createDataFrameCache(): DfCache {
+  let df: IDataFrame | undefined;
+  const cache: DfCache = {
+    get: () => {
+      return df;
+    },
+    set: (prom: IDataFrame) => {
+      df = prom;
+      return prom;
+    },
+    clear: () => {
+      df = undefined;
+    },
+  };
+  return cache;
+}
diff --git a/src/plugins/data/common/data_frames/fields/index.ts b/src/plugins/data/common/data_frames/fields/index.ts
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+export * from './types';
diff --git a/src/plugins/data/common/data_frames/fields/types.ts b/src/plugins/data/common/data_frames/fields/types.ts
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+export interface IFieldType {
+  name: string;
+  type: string;
+  values: any[];
+  count?: number;
+  aggregatable?: boolean;
+  filterable?: boolean;
+  searchable?: boolean;
+  sortable?: boolean;
+  visualizable?: boolean;
+  displayName?: string;
+  format?: any;
+}
diff --git a/src/plugins/data/common/data_frames/index.ts b/src/plugins/data/common/data_frames/index.ts
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+export * from './types';
+export * from './utils';
diff --git a/src/plugins/data/common/data_frames/types.ts b/src/plugins/data/common/data_frames/types.ts
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import { IFieldType } from './fields';
+
+export * from './_df_cache';
+
+export interface IDataFrame {
+  name?: string;
+  schema?: Array<Partial<IFieldType>>;
+  fields: IFieldType[];
+  size: number;
+}
+
+export interface DataFrameAgg {
+  key: string;
+  value: number;
+}
+
+export interface PartialDataFrame extends Omit<IDataFrame, 'fields' | 'size'> {
+  fields: Array<Partial<IFieldType>>;
+}
+
+/**
+ * To be utilize with aggregations and will map to buckets
+ * Plugins can get the aggreted value by their own logic
+ * Setting to null will disable the aggregation if plugin wishes
+ * In future, if the plugin doesn't intentionally set the value to null,
+ * we can calculate the value based on the fields.
+ */
+export interface IDataFrameWithAggs extends IDataFrame {
+  aggs: DataFrameAgg[] | null;
+}