[2.x][Manual Backport ] Back port main to 2.x

.eslintignore

@@ -30,3 +30,7 @@ target
 /packages/osd-test/src/functional_test_runner/lib/config/__tests__/fixtures/
 /packages/osd-ui-framework/dist
 /packages/osd-ui-shared-deps/flot_charts
+
+# antlr overrides
+/src/plugins/data/public/antlr/opensearch_sql/.generated/*
+/src/plugins/data/public/antlr/opensearch_sql/grammar/**/*

.eslintrc.js

@@ -751,5 +751,14 @@ module.exports = {
         'no-undef': 'off',
       },
     },
+    {
+      files: [
+        'src/plugins/data/public/antlr/opensearch_sql/.generated/*',
+        'src/plugins/data/public/antlr/opensearch_sql/grammar/**/*',
+      ],
+      rules: {
+        'filenames/match-regex': 'off',
+      },
+    },
   ],
 };

changelogs/fragments/7336.yml

@@ -0,0 +1,2 @@
+feat:
+- [Auto Suggest] OpenSearch SQL autosuggest with ANTLR ([#7336](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7336))

changelogs/fragments/7363.yml

@@ -0,0 +1,2 @@
+feat:
+- Bump OUI to 1.8.0 ([#7363](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7363))

changelogs/fragments/7368.yml

@@ -0,0 +1,2 @@
+feat:
+- [Discover] Adds a dataset selector for Discover ([#7368](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7368))

changelogs/fragments/7401.yml

@@ -0,0 +1,2 @@
+fix:
+- Make breadcrumb of 4 new added applications comply with BrowserRouter. ([#7401](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7401))

changelogs/fragments/7435.yml

@@ -0,0 +1,2 @@
+fix:
+- [Bug][Workspace] Add permission validation at workspace detail page ([#7435](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7435))

changelogs/fragments/7440.yml

@@ -0,0 +1,2 @@
+fix:
+- [Bug][Data Source] Move data source manageable feature flag to DSM plugin ([#7440](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7440))

config/opensearch_dashboards.yml

@@ -325,7 +325,7 @@
 # "none": The data source is readonly for all users.
 # "dashboard_admin": The data source can only be managed by dashboard admin.
 # "all": The data source can be managed by all users. Default to "all".
-# data_source.manageableBy: "all"
+# data_source_management.manageableBy: "all"
 
 # Set the value of this setting to false to hide the help menu link to the OpenSearch Dashboards user survey
 # opensearchDashboards.survey.url: "https://survey.opensearch.org"
@@ -362,4 +362,4 @@
 # Set the backend roles in groups or users, whoever has the backend roles or exactly match the user ids defined in this config will be regard as dashboard admin.
 # Dashboard admin will have the access to all the workspaces(workspace.enabled: true) and objects inside OpenSearch Dashboards.
 # opensearchDashboards.dashboardAdmin.groups: ["dashboard_admin"]
-# opensearchDashboards.dashboardAdmin.users: ["dashboard_admin"]
+# opensearchDashboards.dashboardAdmin.users: ["dashboard_admin"]

package.json

@@ -63,7 +63,7 @@
     "start": "scripts/use_node scripts/opensearch_dashboards --dev",
     "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST",
     "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security",
-    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true",
+    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true",
     "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev",
     "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev",
     "lint": "yarn run lint:es && yarn run lint:style",
@@ -77,7 +77,12 @@
     "osd:bootstrap": "scripts/use_node scripts/build_ts_refs && scripts/use_node scripts/register_git_hook",
     "spec_to_console": "scripts/use_node scripts/spec_to_console",
     "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"",
-    "release_note:generate": "scripts/use_node scripts/generate_release_note"
+    "release_note:generate": "scripts/use_node scripts/generate_release_note",
+    "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false",
+    "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500",
+    "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"",
+    "osd:ciGroup11": "echo \"apps/vis_builder/*.js\"",
+    "generate:opensearchsqlantlr": "./node_modules/antlr4ng-cli/index.js -Dlanguage=TypeScript -o ./src/plugins/data/public/antlr/opensearch_sql/.generated -visitor -no-listener -Xexact-output-dir ./src/plugins/data/public/antlr/opensearch_sql/grammar/OpenSearchSQLLexer.g4 ./src/plugins/data/public/antlr/opensearch_sql/grammar/OpenSearchSQLParser.g4"
   },
   "repository": {
     "type": "git",
@@ -109,7 +114,7 @@
     "**/qs": "^6.11.0",
     "**/semver": "^7.5.3",
     "**/set-value": "^4.1.0",
-    "**/tar":"^6.2.1",
+    "**/tar": "^6.2.1",
     "**/topo/hoek": "npm:@amoo-miki/hoek@6.1.3",
     "**/trim": "^0.0.3",
     "**/typescript": "4.0.2",
@@ -140,7 +145,7 @@
   "dependencies": {
     "@aws-crypto/client-node": "^3.1.1",
     "@elastic/datemath": "5.0.3",
-    "@elastic/eui": "npm:@opensearch-project/oui@1.7.0",
+    "@elastic/eui": "npm:@opensearch-project/oui@1.8.0",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0",
     "@elastic/request-crypto": "2.0.0",
@@ -177,6 +182,8 @@
     "JSONStream": "1.3.5",
     "abortcontroller-polyfill": "^1.4.0",
     "ajv": "^8.11.0",
+    "antlr4-c3": "^3.4.1",
+    "antlr4ng": "^3.0.4",
     "bluebird": "3.5.5",
     "chalk": "^4.1.0",
     "chokidar": "^3.4.2",
@@ -242,11 +249,13 @@
     "type-detect": "^4.0.8",
     "uuid": "3.3.2",
     "whatwg-fetch": "^3.0.0",
-    "yauzl": "^2.10.0"
+    "yauzl": "^2.10.0",
+    "@opensearch-dashboards-test/opensearch-dashboards-test-library": "https://github.com/opensearch-project/opensearch-dashboards-test-library/archive/refs/tags/1.0.6.tar.gz"
   },
   "devDependencies": {
     "@babel/core": "^7.22.9",
     "@babel/parser": "^7.22.9",
+    "@babel/plugin-transform-class-static-block": "^7.24.4",
     "@babel/register": "^7.22.9",
     "@babel/types": "^7.22.9",
     "@elastic/apm-rum": "^5.6.1",
@@ -361,6 +370,7 @@
     "@types/zen-observable": "^0.8.0",
     "@typescript-eslint/eslint-plugin": "^3.10.0",
     "@typescript-eslint/parser": "^3.10.0",
+    "antlr4ng-cli": "^2.0.0",
     "archiver": "^5.3.0",
     "axe-core": "^4.0.2",
     "babel-eslint": "^10.0.3",
@@ -461,6 +471,7 @@
     "regenerate": "^1.4.0",
     "reselect": "^4.0.0",
     "resize-observer-polyfill": "^1.5.1",
+    "rimraf": "^5.0.7",
     "selenium-webdriver": "^4.0.0-alpha.7",
     "simple-git": "^3.16.0",
     "sinon": "^7.4.2",

packages/osd-babel-preset/webpack_preset.js

@@ -55,6 +55,12 @@ module.exports = () => {
           fileName: false,
         },
       ],
+      [
+        require.resolve('@babel/plugin-transform-class-static-block'),
+        {
+          fileName: false,
+        },
+      ],
     ],
     env: {
       production: {

packages/osd-monaco/src/xjson/lexer_rules/index.ts

@@ -33,6 +33,7 @@ import * as monaco from 'monaco-editor/esm/vs/editor/editor.api';
 import * as xJson from './xjson';
 import * as opensearchql from './opensearchql';
 import * as painless from './painless';
+import * as opensearchsql from './opensearchsql';
 
 export const registerLexerRules = (m: typeof monaco) => {
   m.languages.register({ id: xJson.ID });
@@ -41,4 +42,6 @@ export const registerLexerRules = (m: typeof monaco) => {
   m.languages.setMonarchTokensProvider(painless.ID, painless.lexerRules);
   m.languages.register({ id: opensearchql.ID });
   m.languages.setMonarchTokensProvider(opensearchql.ID, opensearchql.lexerRules);
+  m.languages.register({ id: opensearchsql.ID });
+  m.languages.setMonarchTokensProvider(opensearchsql.ID, opensearchsql.lexerRules);
 };

packages/osd-monaco/src/xjson/lexer_rules/opensearchsql.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { monaco } from '../../monaco';
+
+export const ID = 'SQL';
+
+const keywords = [
+  'ALL',
+  'AND',
+  'AS',
+  'ASC',
+  'BOOLEAN',
+  'BETWEEN',
+  'BY',
+  'CASE',
+  'CAST',
+  'CROSS',
+  'COLUMNS',
+  'DATETIME',
+  'DELETE',
+  'DESC',
+  'DESCRIBE',
+  'DISTINCT',
+  'DOUBLE',
+  'ELSE',
+  'EXISTS',
+  'FALSE',
+  'FLOAT',
+  'FIRST',
+  'FROM',
+  'GROUP',
+  'HAVING',
+  'IN',
+  'INNER',
+  'INT',
+  'INTEGER',
+  'IS',
+  'JOIN',
+  'LAST',
+  'LEFT',
+  'LIKE',
+  'LIMIT',
+  'LONG',
+  'MATCH',
+  'NATURAL',
+  'NOT',
+  'NULL',
+  'NULLS',
+  'ON',
+  'OR',
+  'ORDER',
+  'OUTER',
+  'OVER',
+  'PARTITION',
+  'REGEXP',
+  'RIGHT',
+  'SELECT',
+  'SHOW',
+  'STRING',
+  'THEN',
+  'TRUE',
+  'UNION',
+  'USING',
+  'WHEN',
+  'WHERE',
+  'EXCEPT',
+];
+
+const functions = [
+  'AVG',
+  'COUNT',
+  'MAX',
+  'MIN',
+  'SUM',
+  'VAR_POP',
+  'VAR_SAMP',
+  'VARIANCE',
+  'STD',
+  'STDDEV',
+  'STDDEV_POP',
+  'STDDEV_SAMP',
+  'SUBSTRING',
+  'TRIM',
+];
+
+const operators = [
+  '=',
+  '>',
+  '<',
+  '!',
+  '~',
+  '\\|',
+  '&',
+  '\\^',
+  '\\*',
+  '/',
+  '%',
+  '\\+',
+  '-',
+  'DIV',
+  'MOD',
+];
+
+const brackets = [
+  { open: '(', close: ')', token: 'delimiter.parenthesis' },
+  { open: '[', close: ']', token: 'delimiter.square' },
+];
+
+export const lexerRules = {
+  defaultToken: 'invalid',
+  ignoreCase: true,
+  tokenPostfix: '',
+  keywords,
+  functions,
+  operators,
+  brackets,
+  tokenizer: {
+    root: [
+      [
+        /[a-zA-Z_$][a-zA-Z0-9_$]*/,
+        {
+          cases: {
+            '@keywords': 'keyword',
+            '@functions': 'function',
+            '@default': 'identifier',
+          },
+        },
+      ],
+      { include: '@whitespace' },
+      [/[()]/, '@brackets'],
+      [new RegExp(operators.join('|')), 'operator'],
+      [/[0-9]+(\.[0-9]+)?/, 'number'],
+      [/'([^'\\]|\\.)*$/, 'string.invalid'], // non-terminated string
+      [/'/, 'string', '@string'],
+      [/"/, 'string', '@string'],
+    ],
+    whitespace: [
+      [/[ \t\r\n]+/, 'white'],
+      [/\/\*/, 'comment', '@comment'],
+      [/--.*$/, 'comment'],
+    ],
+    string: [
+      [/[^'\\]+/, 'string'],
+      [/\\./, 'string.escape'],
+      [/'/, 'string', '@pop'],
+      [/"/, 'string', '@pop'],
+    ],
+    comment: [
+      [/[^/*]+/, 'comment'],
+      [/\*\//, 'comment', '@pop'],
+      [/[\/*]/, 'comment'],
+    ],
+  },
+} as monaco.languages.IMonarchLanguage;

packages/osd-optimizer/src/worker/webpack.config.ts

@@ -256,6 +256,16 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker:
             loader: 'raw-loader',
           },
         },
+        {
+          test: /\.cjs$/,
+          include: /node_modules/,
+          use: {
+            loader: 'babel-loader',
+            options: {
+              presets: [BABEL_PRESET_PATH],
+            },
+          },
+        },
       ],
     },
 

packages/osd-ui-framework/package.json

@@ -23,7 +23,7 @@
     "enzyme-adapter-react-16": "^1.9.1"
   },
   "devDependencies": {
-    "@elastic/eui": "npm:@opensearch-project/oui@1.7.0",
+    "@elastic/eui": "npm:@opensearch-project/oui@1.8.0",
     "@osd/babel-preset": "1.0.0",
     "@osd/optimizer": "1.0.0",
     "comment-stripper": "^0.0.4",

packages/osd-ui-shared-deps/package.json

@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "@elastic/charts": "31.1.0",
-    "@elastic/eui": "npm:@opensearch-project/oui@1.7.0",
+    "@elastic/eui": "npm:@opensearch-project/oui@1.8.0",
     "@elastic/numeral": "npm:@amoo-miki/numeral@2.6.0",
     "@opensearch/datemath": "5.0.3",
     "@osd/i18n": "1.0.0",

src/core/public/application/types.ts

@@ -539,6 +539,8 @@ export interface AppMountParameters<HistoryLocationState = unknown> {
    * Optional datasource id to pass while mounting app
    */
   dataSourceId?: string;
+
+  optionalRef?: Record<string, React.RefObject<HTMLDivElement>>;
 }
 
 /**