Merge branch 'main' of https://github.com/opensearch-project/OpenSearch…

… into createpit
opensearch-project · May 17, 2022 · 47bb3e7 · 47bb3e7
2 parents 471a64a + c470580
commit 47bb3e7
Show file tree

Hide file tree

Showing 1,038 changed files with 7,727 additions and 1,838 deletions.
diff --git a/build.gradle b/build.gradle
@@ -49,7 +49,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.5.2" apply false
-  id "org.gradle.test-retry" version "1.3.2" apply false
+  id "org.gradle.test-retry" version "1.4.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java
@@ -54,7 +54,7 @@ public class Jdk implements Buildable, Iterable<File> {
         Arrays.asList("darwin", "freebsd", "linux", "mac", "windows")
     );
     private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
-    private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
+    private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final Configuration configuration;

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java
@@ -110,22 +110,38 @@ private void setupRepository(Project project, Jdk jdk) {
 
         if (jdk.getVendor().equals(VENDOR_ADOPTIUM)) {
             repoUrl = "https://github.com/adoptium/temurin" + jdk.getMajor() + "-binaries/releases/download/";
-            // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17).
-            // To distinguish between those, the GA releases have only major version component (fe 17+32),
-            // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
-            // version separator '.' should be enough.
-            artifactPattern = "jdk-"
-                + jdk.getBaseVersion()
-                + "+"
-                + jdk.getBuild()
-                + "/OpenJDK"
-                + jdk.getMajor()
-                + (jdk.getBaseVersion().contains(".") ? "U" : "")
-                + "-jdk_[classifier]_[module]_hotspot_"
-                + jdk.getBaseVersion()
-                + "_"
-                + jdk.getBuild()
-                + ".[ext]";
+
+            if (jdk.getMajor().equals("8")) {
+                // JDK-8 updates are always suffixed with 'U' (fe OpenJDK8U).
+                artifactPattern = "jdk"
+                    + jdk.getBaseVersion()
+                    + "-"
+                    + jdk.getBuild()
+                    + "/OpenJDK"
+                    + jdk.getMajor()
+                    + "U"
+                    + "-jdk_[classifier]_[module]_hotspot_"
+                    + jdk.getBaseVersion()
+                    + jdk.getBuild()
+                    + ".[ext]";
+            } else {
+                // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17).
+                // To distinguish between those, the GA releases have only major version component (fe 17+32),
+                // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
+                // version separator '.' should be enough.
+                artifactPattern = "jdk-"
+                    + jdk.getBaseVersion()
+                    + "+"
+                    + jdk.getBuild()
+                    + "/OpenJDK"
+                    + jdk.getMajor()
+                    + (jdk.getBaseVersion().contains(".") ? "U" : "")
+                    + "-jdk_[classifier]_[module]_hotspot_"
+                    + jdk.getBaseVersion()
+                    + "_"
+                    + jdk.getBuild()
+                    + ".[ext]";
+            }
         } else if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) {
             repoUrl = "https://api.adoptopenjdk.net/v3/binary/version/";
             if (jdk.getMajor().equals("8")) {

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -75,8 +75,8 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "8u242+b08";
-    private static final String SYSTEM_JDK_VENDOR = "adoptopenjdk";
+    private static final String SYSTEM_JDK_VERSION = "11.0.15+10";
+    private static final String SYSTEM_JDK_VENDOR = "adoptium";
     private static final String GRADLE_JDK_VERSION = "17.0.3+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 

diff --git a/client/sniffer/src/test/resources/create_test_nodes_info.bash b/client/sniffer/src/test/resources/create_test_nodes_info.bash
@@ -14,7 +14,7 @@
 # when we do rebuild the files they don't jump around too much. That
 # way the diffs are smaller.
 
-set -e
+set -e -o pipefail
 
 script_path="$( cd "$(dirname "$0")" ; pwd -P )"
 work=$(mktemp -d)

diff --git a/dev-tools/atomic_push.sh b/dev-tools/atomic_push.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -e -o pipefail
 
 if [ "$#" -eq 0 ]; then
     printf 'Usage: %s <origin> <branch> <branch> ...\n' "$(basename "$0")"

diff --git a/dev-tools/signoff-check.sh b/dev-tools/signoff-check.sh
@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
+set -e -o pipefail
 
 ### Script to check for signoff presents on commits
 

diff --git a/distribution/docker/docker-test-entrypoint.sh b/distribution/docker/docker-test-entrypoint.sh
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+set -e -o pipefail
+
 cd /usr/share/opensearch/bin/
 
 /usr/local/bin/docker-entrypoint.sh | tee > /usr/share/opensearch/logs/console.log
diff --git a/distribution/docker/src/docker/bin/docker-entrypoint.sh b/distribution/docker/src/docker/bin/docker-entrypoint.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -e
+set -e -o pipefail
 
 # Files created by Elasticsearch should always be group writable too
 umask 0002

diff --git a/distribution/packages/src/common/scripts/preinst b/distribution/packages/src/common/scripts/preinst
@@ -10,6 +10,8 @@
 #       $1=1       : indicates an new install
 #       $1=2       : indicates an upgrade
 
+set -e -o pipefail
+
 err_exit() {
     echo "$@" >&2
     exit 1

diff --git a/distribution/packages/src/common/systemd/systemd-entrypoint b/distribution/packages/src/common/systemd/systemd-entrypoint
@@ -3,6 +3,8 @@
 # This wrapper script allows SystemD to feed a file containing a passphrase into
 # the main OpenSearch startup script
 
+set -e -o pipefail
+
 if [ -n "$OPENSEARCH_KEYSTORE_PASSPHRASE_FILE" ] ; then
   exec /usr/share/opensearch/bin/opensearch "$@" < "$OPENSEARCH_KEYSTORE_PASSPHRASE_FILE"
 else

diff --git a/distribution/packages/src/deb/init.d/opensearch b/distribution/packages/src/deb/init.d/opensearch
@@ -12,6 +12,8 @@
 # Description:       Starts opensearch using start-stop-daemon
 ### END INIT INFO
 
+set -e -o pipefail
+
 PATH=/bin:/usr/bin:/sbin:/usr/sbin
 NAME=opensearch
 DESC="OpenSearch Server"

diff --git a/distribution/packages/src/rpm/init.d/opensearch b/distribution/packages/src/rpm/init.d/opensearch
@@ -16,6 +16,8 @@
 # Description: OpenSearch is a very scalable, schema-free and high-performance search solution supporting multi-tenancy and near realtime search.
 ### END INIT INFO
 
+set -e -o pipefail
+
 #
 # init.d / servicectl compatibility (openSUSE)
 #

diff --git a/distribution/src/bin/opensearch b/distribution/src/bin/opensearch
@@ -13,6 +13,8 @@
 #
 #   OPENSEARCH_JAVA_OPTS="-Xms8g -Xmx8g" ./bin/opensearch
 
+set -e -o pipefail
+
 source "`dirname "$0"`"/opensearch-env
 
 CHECK_KEYSTORE=true

diff --git a/distribution/src/bin/opensearch-keystore b/distribution/src/bin/opensearch-keystore
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e -o pipefail
+
 OPENSEARCH_MAIN_CLASS=org.opensearch.common.settings.KeyStoreCli \
   OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/keystore-cli \
   "`dirname "$0"`"/opensearch-cli \

diff --git a/distribution/src/bin/opensearch-node b/distribution/src/bin/opensearch-node
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -e -o pipefail
 
 OPENSEARCH_MAIN_CLASS=org.opensearch.cluster.coordination.NodeToolCli \
   "`dirname "$0"`"/opensearch-cli \

diff --git a/distribution/src/bin/opensearch-plugin b/distribution/src/bin/opensearch-plugin
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e -o pipefail
+
 OPENSEARCH_MAIN_CLASS=org.opensearch.plugins.PluginCli \
   OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/plugin-cli \
   "`dirname "$0"`"/opensearch-cli \

diff --git a/distribution/src/bin/opensearch-shard b/distribution/src/bin/opensearch-shard
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e -o pipefail
+
 OPENSEARCH_MAIN_CLASS=org.opensearch.index.shard.ShardToolCli \
   "`dirname "$0"`"/opensearch-cli \
   "$@"
diff --git a/distribution/src/bin/opensearch-upgrade b/distribution/src/bin/opensearch-upgrade
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -e -o pipefail
 
 OPENSEARCH_MAIN_CLASS=org.opensearch.upgrade.UpgradeCli \
   OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/upgrade-cli \

diff --git a/gradle/missing-javadoc.gradle b/gradle/missing-javadoc.gradle
@@ -178,8 +178,7 @@ configure([
 
 configure(project(":server")) {
   project.tasks.withType(MissingJavadocTask) {
-    isExcluded = true
-    // TODO: reenable after fixing missing javadocs
+    // TODO: bump to variable missing level after increasing javadoc coverage
     javadocMissingLevel = "class"
   }
 }

diff --git a/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml
@@ -7,10 +7,10 @@
     - do:
         cluster.state: {}
 
-    # Get master node id
-    - set: { master_node: master }
+    # Get cluster-manager node id
+    - set: { cluster_manager_node: cluster_manager }
 
     - do:
         nodes.info: {}
 
-    - contains:  { nodes.$master.modules: { name: aggs-matrix-stats } }
+    - contains:  { nodes.$cluster_manager.modules: { name: aggs-matrix-stats } }
diff --git a/...nalysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/10_basic.yml b/...nalysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/10_basic.yml
@@ -5,10 +5,10 @@
     - do:
         cluster.state: {}
 
-    # Get master node id
-    - set: { master_node: master }
+    # Get cluster-manager node id
+    - set: { cluster_manager_node: cluster_manager }
 
     - do:
         nodes.info: {}
 
-    - contains:  { nodes.$master.modules: { name: analysis-common } }
+    - contains:  { nodes.$cluster_manager.modules: { name: analysis-common } }
diff --git a/...est-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java b/...est-common/src/internalClusterTest/java/org/opensearch/ingest/common/IngestRestartIT.java
@@ -86,7 +86,7 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
 
     public void testFailureInConditionalProcessor() {
         internalCluster().ensureAtLeastNumDataNodes(1);
-        internalCluster().startMasterOnlyNode();
+        internalCluster().startClusterManagerOnlyNode();
         final String pipelineId = "foo";
         client().admin()
             .cluster()

diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml
@@ -5,34 +5,34 @@
     - do:
         cluster.state: {}
 
-    # Get master node id
-    - set: { master_node: master }
+    # Get cluster-manager node id
+    - set: { cluster_manager_node: cluster_manager }
 
     - do:
         nodes.info: {}
 
-    - contains:  { nodes.$master.modules: { name: ingest-common } }
-    - contains:  { nodes.$master.ingest.processors: { type: append } }
-    - contains:  { nodes.$master.ingest.processors: { type: bytes } }
-    - contains:  { nodes.$master.ingest.processors: { type: convert } }
-    - contains:  { nodes.$master.ingest.processors: { type: date } }
-    - contains:  { nodes.$master.ingest.processors: { type: date_index_name } }
-    - contains:  { nodes.$master.ingest.processors: { type: dissect } }
-    - contains:  { nodes.$master.ingest.processors: { type: dot_expander } }
-    - contains:  { nodes.$master.ingest.processors: { type: fail } }
-    - contains:  { nodes.$master.ingest.processors: { type: foreach } }
-    - contains:  { nodes.$master.ingest.processors: { type: grok } }
-    - contains:  { nodes.$master.ingest.processors: { type: gsub } }
-    - contains:  { nodes.$master.ingest.processors: { type: html_strip } }
-    - contains:  { nodes.$master.ingest.processors: { type: join } }
-    - contains:  { nodes.$master.ingest.processors: { type: json } }
-    - contains:  { nodes.$master.ingest.processors: { type: kv } }
-    - contains:  { nodes.$master.ingest.processors: { type: lowercase } }
-    - contains:  { nodes.$master.ingest.processors: { type: remove } }
-    - contains:  { nodes.$master.ingest.processors: { type: rename } }
-    - contains:  { nodes.$master.ingest.processors: { type: script } }
-    - contains:  { nodes.$master.ingest.processors: { type: set } }
-    - contains:  { nodes.$master.ingest.processors: { type: sort } }
-    - contains:  { nodes.$master.ingest.processors: { type: split } }
-    - contains:  { nodes.$master.ingest.processors: { type: trim } }
-    - contains:  { nodes.$master.ingest.processors: { type: uppercase } }
+    - contains:  { nodes.$cluster_manager.modules: { name: ingest-common } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: append } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: bytes } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: convert } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: date } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: date_index_name } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: dissect } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: dot_expander } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: fail } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: foreach } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: grok } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: gsub } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: html_strip } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: join } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: json } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: kv } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: lowercase } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: remove } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: rename } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: script } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: set } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: sort } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: split } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: trim } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: uppercase } }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
@@ -77,21 +77,21 @@ teardown:
 
   - do:
       cluster.state: {}
-    # Get master node id
-  - set: { master_node: master }
+    # Get cluster-manager node id
+  - set: { cluster_manager_node: cluster_manager }
 
   - do:
       nodes.stats:
         metric: [ ingest ]
   #we can't assert anything here since we might have more than one node in the cluster
-  - gte: {nodes.$master.ingest.total.count: 0}
-  - gte: {nodes.$master.ingest.total.failed: 0}
-  - gte: {nodes.$master.ingest.total.time_in_millis: 0}
-  - match: {nodes.$master.ingest.total.current: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline1.count: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline1.failed: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline1.time_in_millis: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline1.current: 0}
+  - gte: {nodes.$cluster_manager.ingest.total.count: 0}
+  - gte: {nodes.$cluster_manager.ingest.total.failed: 0}
+  - gte: {nodes.$cluster_manager.ingest.total.time_in_millis: 0}
+  - match: {nodes.$cluster_manager.ingest.total.current: 0}
+  - gte: {nodes.$cluster_manager.ingest.pipelines.pipeline1.count: 0}
+  - match: {nodes.$cluster_manager.ingest.pipelines.pipeline1.failed: 0}
+  - gte: {nodes.$cluster_manager.ingest.pipelines.pipeline1.time_in_millis: 0}
+  - match: {nodes.$cluster_manager.ingest.pipelines.pipeline1.current: 0}
 
 ---
 "Test bulk request with default pipeline":
@@ -113,21 +113,21 @@ teardown:
 
   - do:
       cluster.state: {}
-  # Get master node id
-  - set: { master_node: master }
+  # Get cluster-manager node id
+  - set: { cluster_manager_node: cluster_manager }
 
   - do:
       nodes.stats:
         metric: [ ingest ]
   #we can't assert anything here since we might have more than one node in the cluster
-  - gte: {nodes.$master.ingest.total.count: 0}
-  - gte: {nodes.$master.ingest.total.failed: 0}
-  - gte: {nodes.$master.ingest.total.time_in_millis: 0}
-  - match: {nodes.$master.ingest.total.current: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline2.count: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline2.failed: 0}
-  - gte: {nodes.$master.ingest.pipelines.pipeline2.time_in_millis: 0}
-  - match: {nodes.$master.ingest.pipelines.pipeline2.current: 0}
+  - gte: {nodes.$cluster_manager.ingest.total.count: 0}
+  - gte: {nodes.$cluster_manager.ingest.total.failed: 0}
+  - gte: {nodes.$cluster_manager.ingest.total.time_in_millis: 0}
+  - match: {nodes.$cluster_manager.ingest.total.current: 0}
+  - gte: {nodes.$cluster_manager.ingest.pipelines.pipeline2.count: 0}
+  - match: {nodes.$cluster_manager.ingest.pipelines.pipeline2.failed: 0}
+  - gte: {nodes.$cluster_manager.ingest.pipelines.pipeline2.time_in_millis: 0}
+  - match: {nodes.$cluster_manager.ingest.pipelines.pipeline2.current: 0}
 
   - do:
       get:

diff --git a/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/opensearch/ingest/geoip/DatabaseReaderLazyLoader.java
@@ -71,7 +71,7 @@ class DatabaseReaderLazyLoader implements Closeable {
 
     /**
      * Read the database type from the database. We do this manually instead of relying on the built-in mechanism to avoid reading the
-     * entire database into memory merely to read the type. This is especially important to maintain on master nodes where pipelines are
+     * entire database into memory merely to read the type. This is especially important to maintain on cluster-manager nodes where pipelines are
      * validated. If we read the entire database into memory, we could potentially run into low-memory constraints on such nodes where
      * loading this data would otherwise be wasteful if they are not also ingest nodes.
      *

diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/10_basic.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/10_basic.yml
@@ -5,10 +5,10 @@
     - do:
         cluster.state: {}
 
-    - set: {master_node: master}
+    - set: { cluster_manager_node: cluster_manager }
 
     - do:
         nodes.info: {}
 
-    - contains:  { nodes.$master.modules: { name: ingest-geoip } }
-    - contains:  { nodes.$master.ingest.processors: { type: geoip } }
+    - contains:  { nodes.$cluster_manager.modules: { name: ingest-geoip } }
+    - contains:  { nodes.$cluster_manager.ingest.processors: { type: geoip } }