Merge branch '2.x' into backport/backport-6443-to-2.x

Signed-off-by: Qingyang(Abby) Hu <abigailhu2000@gmail.com>

.lycheeignore

@@ -88,3 +88,4 @@ https://unpkg.com/@elastic/
 https://codeload.github.com/
 https://www.quandl.com/api/v1/datasets/
 https://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+site.com

changelogs/fragments/5652.yml

@@ -0,0 +1,2 @@
+feat:
+- Make theme and dark mode settings user/device specific (in local storage), with opt-out ([#5652](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5652))

changelogs/fragments/6613.yml

@@ -0,0 +1,2 @@
+feat:
+- Support language selector from the data plugin ([#6613](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6613))

changelogs/fragments/6678.yml

@@ -0,0 +1,2 @@
+fix:
+- [MDS] Add a new message to data source components when there are no compatible datasources ([#6678](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6678))

changelogs/fragments/6729.yml

@@ -0,0 +1,2 @@
+fix:
+- Add more test for icon and aggregated view ([#6729](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6729))

changelogs/fragments/6733.yml

@@ -0,0 +1,2 @@
+fix:
+- [OSD Availability] Prevent OSD process crashes when disk is full ([#6733](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6733))

changelogs/fragments/6742.yml

@@ -0,0 +1,2 @@
+fix:
+- Add test for edit data source form ([#6742](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6742))

changelogs/fragments/6759.yml

@@ -0,0 +1,2 @@
+feat:
+- Make Field Name Search Filter Case Insensitive ([#6759](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6759))

changelogs/fragments/6771.yml

@@ -0,0 +1,2 @@
+fix:
+- Lint checker failure fix ([#6771](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6771))

config/opensearch_dashboards.yml

@@ -135,6 +135,11 @@
 # Enables you to specify a file where OpenSearch Dashboards stores log output.
 #logging.dest: stdout
 
+# This configuration option controls the handling of error messages in the logging stream. It is disabled by default.
+# When set to true, the 'ENOSPC' error message will not cause the OpenSearch Dashboards process to crash. Otherwise,
+# the original behavior will be maintained.
+#logging.ignoreEnospcError: false
+
 # Set the value of this setting to true to suppress all logging output.
 #logging.silent: false
 
@@ -326,3 +331,6 @@
 # Set the value to true to enable Ui Metric Collectors in Usage Collector
 # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false
 # usageCollection.uiMetric.enabled: false
+
+# Set the value to true to enable enhancements for the data plugin
+# data.enhancements.enabled: false

package.json

@@ -152,6 +152,7 @@
     "@hapi/wreck": "^17.1.0",
     "@opensearch-project/opensearch": "^1.1.0",
     "@opensearch-project/opensearch-next": "npm:@opensearch-project/opensearch@^2.6.0",
+    "@opensearch/datemath": "5.0.3",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",

packages/opensearch-datemath/index.d.ts

@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

packages/opensearch-datemath/index.js

@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d, momentInstance = moment) => momentInstance.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text, momentInstance)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

packages/opensearch-datemath/index.test.ts

@@ -122,19 +122,19 @@ describe('dateMath', function () {
     });
 
     it('should return a moment if passed a date', function () {
-      expect(dateMath.parse(date).format(format)).to.eql(mmnt.format(format));
+      expect(dateMath.parse(date)!.format(format)).to.eql(mmnt.format(format));
     });
 
     it('should return a moment if passed an ISO8601 string', function () {
-      expect(dateMath.parse(string).format(format)).to.eql(mmnt.format(format));
+      expect(dateMath.parse(string)!.format(format)).to.eql(mmnt.format(format));
     });
 
     it('should return the current time when parsing now', function () {
-      expect(dateMath.parse('now').format(format)).to.eql(now.format(format));
+      expect(dateMath.parse('now')!.format(format)).to.eql(now.format(format));
     });
 
     it('should use the forceNow parameter when parsing now', function () {
-      expect(dateMath.parse('now', { forceNow: anchoredDate }).valueOf()).to.eql(unix);
+      expect(dateMath.parse('now', { forceNow: anchoredDate })!.valueOf()).to.eql(unix);
     });
   });
 
@@ -158,17 +158,17 @@ describe('dateMath', function () {
         const thenEx = `${anchor}||-${len}${span}`;
 
         it('should return ' + len + span + ' ago', function () {
-          const parsed = dateMath.parse(nowEx).format(format);
+          const parsed = dateMath.parse(nowEx)!.format(format);
           expect(parsed).to.eql(now.subtract(len, span).format(format));
         });
 
         it('should return ' + len + span + ' before ' + anchor, function () {
-          const parsed = dateMath.parse(thenEx).format(format);
+          const parsed = dateMath.parse(thenEx)!.format(format);
           expect(parsed).to.eql(anchored.subtract(len, span).format(format));
         });
 
         it('should return ' + len + span + ' before forceNow', function () {
-          const parsed = dateMath.parse(nowEx, { forceNow: anchoredDate }).valueOf();
+          const parsed = dateMath.parse(nowEx, { forceNow: anchoredDate })!.valueOf();
           expect(parsed).to.eql(anchored.subtract(len, span).valueOf());
         });
       });
@@ -195,17 +195,17 @@ describe('dateMath', function () {
         const thenEx = `${anchor}||+${len}${span}`;
 
         it('should return ' + len + span + ' from now', function () {
-          expect(dateMath.parse(nowEx).format(format)).to.eql(now.add(len, span).format(format));
+          expect(dateMath.parse(nowEx)!.format(format)).to.eql(now.add(len, span).format(format));
         });
 
         it('should return ' + len + span + ' after ' + anchor, function () {
-          expect(dateMath.parse(thenEx).format(format)).to.eql(
+          expect(dateMath.parse(thenEx)!.format(format)).to.eql(
             anchored.add(len, span).format(format)
           );
         });
 
         it('should return ' + len + span + ' after forceNow', function () {
-          expect(dateMath.parse(nowEx, { forceNow: anchoredDate }).valueOf()).to.eql(
+          expect(dateMath.parse(nowEx, { forceNow: anchoredDate })!.valueOf()).to.eql(
             anchored.add(len, span).valueOf()
           );
         });
@@ -229,26 +229,26 @@ describe('dateMath', function () {
 
     spans.forEach((span) => {
       it(`should round now to the beginning of the ${span}`, function () {
-        expect(dateMath.parse('now/' + span).format(format)).to.eql(
+        expect(dateMath.parse('now/' + span)!.format(format)).to.eql(
           now.startOf(span).format(format)
         );
       });
 
       it(`should round now to the beginning of forceNow's ${span}`, function () {
-        expect(dateMath.parse('now/' + span, { forceNow: anchoredDate }).valueOf()).to.eql(
+        expect(dateMath.parse('now/' + span, { forceNow: anchoredDate })!.valueOf()).to.eql(
           anchored.startOf(span).valueOf()
         );
       });
 
       it(`should round now to the end of the ${span}`, function () {
-        expect(dateMath.parse('now/' + span, { roundUp: true }).format(format)).to.eql(
+        expect(dateMath.parse('now/' + span, { roundUp: true })!.format(format)).to.eql(
           now.endOf(span).format(format)
         );
       });
 
       it(`should round now to the end of forceNow's ${span}`, function () {
         expect(
-          dateMath.parse('now/' + span, { roundUp: true, forceNow: anchoredDate }).valueOf()
+          dateMath.parse('now/' + span, { roundUp: true, forceNow: anchoredDate })!.valueOf()
         ).to.eql(anchored.endOf(span).valueOf());
       });
     });
@@ -269,28 +269,28 @@ describe('dateMath', function () {
     });
 
     it('should round to the nearest second with 0 value', function () {
-      const val = dateMath.parse('now-0s/s').format(format);
+      const val = dateMath.parse('now-0s/s')!.format(format);
       expect(val).to.eql(now.startOf('s').format(format));
     });
 
     it('should subtract 17s, rounded to the nearest second', function () {
-      const val = dateMath.parse('now-17s/s').format(format);
+      const val = dateMath.parse('now-17s/s')!.format(format);
       expect(val).to.eql(now.startOf('s').subtract(17, 's').format(format));
     });
 
     it('should add 555ms, rounded to the nearest millisecond', function () {
-      const val = dateMath.parse('now+555ms/ms').format(format);
+      const val = dateMath.parse('now+555ms/ms')!.format(format);
       expect(val).to.eql(now.add(555, 'ms').startOf('ms').format(format));
     });
 
     it('should subtract 555ms, rounded to the nearest second', function () {
-      const val = dateMath.parse('now-555ms/s').format(format);
+      const val = dateMath.parse('now-555ms/s')!.format(format);
       expect(val).to.eql(now.subtract(555, 'ms').startOf('s').format(format));
     });
 
     it('should round weeks to Sunday by default', function () {
       const val = dateMath.parse('now-1w/w');
-      expect(val.isoWeekday()).to.eql(7);
+      expect(val!.isoWeekday()).to.eql(7);
     });
 
     it('should round weeks based on the passed moment locale start of week setting', function () {
@@ -300,7 +300,7 @@ describe('dateMath', function () {
         week: { dow: 2 },
       });
       const val = dateMath.parse('now-1w/w', { momentInstance: m });
-      expect(val.isoWeekday()).to.eql(2);
+      expect(val!.isoWeekday()).to.eql(2);
     });
 
     it('should round up weeks based on the passed moment locale start of week setting', function () {
@@ -315,11 +315,11 @@ describe('dateMath', function () {
       });
       // The end of the range (rounding up) should be the last day of the week (so one day before)
       // our start of the week, that's why 3 - 1
-      expect(val.isoWeekday()).to.eql(3 - 1);
+      expect(val!.isoWeekday()).to.eql(3 - 1);
     });
 
     it('should round relative to forceNow', function () {
-      const val = dateMath.parse('now-0s/s', { forceNow: anchoredDate }).valueOf();
+      const val = dateMath.parse('now-0s/s', { forceNow: anchoredDate })!.valueOf();
       expect(val).to.eql(anchored.startOf('s').valueOf());
     });
 

packages/osd-opensearch/src/cli_commands/snapshot.js

@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

packages/osd-opensearch/src/cluster.js

@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *