[AUTO] Increment version to 2.11.0.0

common/constants/index.ts

@@ -5,11 +5,19 @@
 
 export const PLUGIN_ID = 'queryWorkbenchDashboards';
 export const PLUGIN_NAME = 'Query Workbench';
-export const OPENSEARCH_ACC_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest';
-export const ACC_INDEX_TYPE_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest';
+export const OPENSEARCH_ACC_DOCUMENTATION_URL =
+  'https://opensearch.org/docs/latest/dashboards/management/accelerate-external-data/';
+export const ACC_INDEX_TYPE_DOCUMENTATION_URL =
+  'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md';
 
-export const SKIPPING_INDEX = `skipping_index`;
-export const ON_LOAD_QUERY = `SHOW tables LIKE '%';`;
+export const TREE_ITEM_SKIPPING_INDEX_DEFAULT_NAME = `skipping_index`;
+export const TREE_ITEM_COVERING_INDEX_DEFAULT_NAME = `covering_index`;
+export const TREE_ITEM_MATERIALIZED_VIEW_DEFAULT_NAME = `materialized_view`;
+export const TREE_ITEM_DATABASE_NAME_DEFAULT_NAME = `database`;
+export const TREE_ITEM_TABLE_NAME_DEFAULT_NAME = `table`;
+export const TREE_ITEM_LOAD_MATERIALIZED_BADGE_NAME = `Load Materialized View`;
+export const TREE_ITEM_BADGE_NAME = `badge`;
+export const LOAD_OPENSEARCH_INDICES_QUERY = `SHOW tables LIKE '%';`;
 export const SKIPPING_INDEX_QUERY = `CREATE SKIPPING INDEX ON myS3.logs_db.http_logs 
 (status VALUE_SET) 
 WITH (
@@ -37,7 +45,7 @@ OPTIONS (
 export const ACCELERATION_INDEX_TYPES = [
   { label: 'Skipping Index', value: 'skipping' },
   { label: 'Covering Index', value: 'covering' },
-  // { label: 'Materialized View', value: 'materialized' }, Hidden Option -> Until opensearch-spark feature is ready
+  { label: 'Materialized View', value: 'materialized' },
 ];
 
 export const ACCELERATION_AGGREGRATION_FUNCTIONS = [
@@ -63,7 +71,7 @@ export const SKIPPING_INDEX_ACCELERATION_METHODS = [
 ];
 
 export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)';
-export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z][a-z_\-]*$/;
+export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z][a-z_]*$/;
 export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+\/.*/;
 export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping';
 
@@ -78,7 +86,12 @@ export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices
 - 'Materialized View' indices also enable users to define their index name, but they do not have a suffix.
   - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`.
 ##### Note:
-- All user given index names must be in lowercase letters. Cannot begin with underscores or hyphens. Spaces, commas, and characters :, ", *, +, /, \, |, ?, #, >, or < are not allowed.  
+- All user given index names must be in lowercase letters. Index name cannot begin with underscores. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed.  
   `;
 
-export const SIDEBAR_POLL_INTERVAL_MS = 5000;
+export const TIMESTAMP_DATATYPE = 'timestamp';
+export const FETCH_OPENSEARCH_INDICES_PATH = '/api/sql_console/sqlquery';
+export const POLL_INTERVAL_MS = 2000;
+export const ASYNC_QUERY_ENDPOINT = '/api/spark_sql_console';
+export const ASYNC_QUERY_JOB_ENDPOINT = ASYNC_QUERY_ENDPOINT + '/job/';
+export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id';

common/types/index.ts

@@ -32,6 +32,11 @@ export interface RefreshIntervalType {
   refreshInterval: string;
 }
 
+export interface watermarkDelayType {
+  delayWindow: number;
+  delayInterval: string;
+}
+
 export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized';
 
 export interface GroupByTumbleType {
@@ -57,8 +62,11 @@ export interface FormErrorsType {
   replicaShardsError: string[];
   refreshIntervalError: string[];
   checkpointLocationError: string[];
+  watermarkDelayError: string[];
 }
 
+export type AccelerationRefreshType = 'auto' | 'interval' | 'manual';
+
 export interface CreateAccelerationForm {
   dataSource: string;
   database: string;
@@ -71,10 +79,32 @@ export interface CreateAccelerationForm {
   accelerationIndexName: string;
   primaryShardsCount: number;
   replicaShardsCount: number;
-  refreshType: 'interval' | 'auto';
+  refreshType: AccelerationRefreshType;
   checkpointLocation: string | undefined;
+  watermarkDelay: watermarkDelayType;
   refreshIntervalOptions: RefreshIntervalType;
   formErrors: FormErrorsType;
 }
 
-export type AsyncQueryLoadingStatus = "SUCCESS" | "FAILED" | "RUNNING" | "SCHEDULED" | "CANCELED"
+export type AsyncQueryLoadingStatus = 'SUCCESS' | 'FAILED' | 'RUNNING' | 'SCHEDULED' | 'CANCELLED';
+export type TreeItemType =
+  | 'covering_index'
+  | 'skipping_index'
+  | 'table'
+  | 'database'
+  | 'materialized_view'
+  | 'Load Materialized View'
+  | 'badge';
+
+export interface TreeItem {
+  name: string;
+  type: TreeItemType;
+  isExpanded: boolean;
+  values?: TreeItem[];
+  isLoading?: boolean;
+}
+
+export interface DatasourceTreeLoading {
+  flag: boolean;
+  status: string;
+}

common/utils/async_query_helpers.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import _ from 'lodash';
+import { CoreStart } from '../../../../src/core/public';
+import {
+  ASYNC_QUERY_ENDPOINT,
+  ASYNC_QUERY_JOB_ENDPOINT,
+  ASYNC_QUERY_SESSION_ID,
+  POLL_INTERVAL_MS,
+} from '../constants';
+
+export const setAsyncSessionId = (value: string | null) => {
+  if (value === null) sessionStorage.removeItem(ASYNC_QUERY_SESSION_ID);
+  else sessionStorage.setItem(ASYNC_QUERY_SESSION_ID, value);
+};
+
+export const getAsyncSessionId = () => {
+  return sessionStorage.getItem(ASYNC_QUERY_SESSION_ID);
+};
+
+export const getJobId = (query: {}, http: CoreStart['http'], callback) => {
+  http
+    .post(ASYNC_QUERY_ENDPOINT, {
+      body: JSON.stringify({ ...query, sessionId: getAsyncSessionId() ?? undefined }),
+    })
+    .then((res) => {
+      const id = res.data.resp.queryId;
+      setAsyncSessionId(_.get(res.data.resp, 'sessionId', null));
+      callback(id);
+    })
+    .catch((err) => {
+      console.error(err);
+    });
+};
+
+export const pollQueryStatus = (id: string, http: CoreStart['http'], callback) => {
+  http
+    .get(ASYNC_QUERY_JOB_ENDPOINT + id)
+    .then((res) => {
+      const status = res.data.resp.status.toLowerCase();
+      if (
+        status === 'pending' ||
+        status === 'running' ||
+        status === 'scheduled' ||
+        status === 'waiting'
+      ) {
+        callback({ status: status });
+        setTimeout(() => pollQueryStatus(id, http, callback), POLL_INTERVAL_MS);
+      } else if (status === 'failed') {
+        callback({ status: 'FAILED', results: [] });
+      } else if (status === 'success') {
+        const results = _.get(res.data.resp, 'datarows');
+        callback({ status: 'SUCCESS', results: results });
+      }
+    })
+    .catch((err) => {
+      console.error(err);
+      callback([]);
+    });
+};

opensearch_dashboards.json

@@ -4,6 +4,8 @@
   "opensearchDashboardsVersion": "2.11.0",
   "server": true,
   "ui": true,
-  "requiredPlugins": ["navigation"],
+  "requiredPlugins": [
+    "navigation"
+  ],
   "optionalPlugins": []
-}
+}

package.json

@@ -47,4 +47,4 @@
     "semver": "^7.5.2",
     "@cypress/request": "^3.0.0"
   }
-}
+}