Merge branch 'main' into xcontentstatuscode

.github/workflows/backport.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   backport:
+    if: ${{ contains(github.event.label.name, 'backport') }}
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -26,4 +27,3 @@ jobs:
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
           head_template: backport/backport-<%= number %>-to-<%= base %>
-          files_to_skip: 'CHANGELOG.md'

CONTRIBUTING.md

@@ -119,20 +119,39 @@ You may type this line on your own when writing your commit messages. However, i
 
 ## Changelog
 
-OpenSearch maintains version specific changelog by enforcing a change to the ongoing [CHANGELOG](CHANGELOG.md) file adhering to the [Keep A Changelog](https://keepachangelog.com/en/1.0.0/) format.
+OpenSearch maintains version specific changelog by enforcing a change to the ongoing [CHANGELOG](CHANGELOG.md) file adhering to the [Keep A Changelog](https://keepachangelog.com/en/1.0.0/) format. The purpose of the changelog is for the contributors and maintainers to incrementally build the release notes throughout the development process to avoid a painful and error-prone process of attempting to compile the release notes at release time. On each release the "unreleased" entries of the changelog are moved to the appropriate release notes document in the `./release-notes` folder. Also, incrementally building the changelog provides a concise, human-readable list of significant features that have been added to the unreleased version under development.
 
-Briefly, the changes are curated by version, with the changes to the main branch added chronologically to `Unreleased` version. Further, each version has corresponding sections which list out the category of the change - `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, `Security`.
+### Which changes require a CHANGELOG entry?
+Changelogs are intended for operators/administrators, developers integrating with libraries and APIs, and end-users interacting with OpenSearch Dashboards and/or the REST API (collectively referred to as "user"). In short, any change that a user of OpenSearch might want to be aware of should be included in the changelog. The changelog is _not_ intended to replace the git commit log that developers of OpenSearch itself rely upon. The following are some examples of changes that should be in the changelog:
 
+- A newly added feature
+- A fix for a user-facing bug
+- Dependency updates
+- Fixes for security issues
 
-### How to add my changes to [CHANGELOG](CHANGELOG.md)?
+The following are some examples where a changelog entry is not necessary:
 
-As a contributor, you must ensure that every pull request has the changes listed out within the corresponding version and appropriate section of [CHANGELOG](CHANGELOG.md) file.
+- Adding, modifying, or fixing tests
+- An incremental PR for a larger feature (such features should include _one_ changelog entry for the feature)
+- Documentation changes or code refactoring
+- Build-related changes
 
-Adding in the change is two step process -
-1. Add your changes to the corresponding section within the CHANGELOG file with dummy pull request information, publish the PR
+Any PR that does not include a changelog entry will result in a failure of the validation workflow in GitHub. If the contributor and maintainers agree that no changelog entry is required, then the `skip-changelog` label can be applied to the PR which will result in the workflow passing.
+
+### How to add my changes to [CHANGELOG](CHANGELOG.md)?
 
+Adding in the change is two step process:
+1. Add your changes to the corresponding section within the CHANGELOG file with dummy pull request information, publish the PR
 2. Update the entry for your change in [`CHANGELOG.md`](CHANGELOG.md) and make sure that you reference the pull request there.
 
+### Where should I put my CHANGELOG entry?
+Please review the [branching strategy](https://github.com/opensearch-project/.github/blob/main/RELEASING.md#opensearch-branching) document. The changelog on the `main` branch will contain sections for the _next major_ and _next minor_ releases. Your entry should go into the section it is intended to be released in. In practice, most changes to `main` will be backported to the next minor release so most entries will likely be in that section.
+
+The following examples assume the _next major_ release on main is 3.0, then _next minor_ release is 2.5, and the _current_ release is 2.4.
+
+- **Add a new feature to release in next minor:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry).
+- **Introduce a breaking API change to release in next major:** Add a changelog entry to `[Unreleased 3.0]` on main, do not backport.
+- **Upgrade a dependency to fix a CVE:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry), then backport to 2.4 and ensure the changelog entry is added to `[Unreleased 2.4.1]`.
 
 ## Review Process
 

build.gradle

@@ -316,6 +316,7 @@ allprojects {
     javadoc.options.encoding = 'UTF8'
     javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
     javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"]
+    javadoc.options.addStringOption("-release", targetCompatibility.majorVersion)
   }
 
   // support for reproducible builds
@@ -412,6 +413,9 @@ gradle.projectsEvaluated {
         if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
           task.jvmArgs += ["-Djava.security.manager=allow"]
         }
+        if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_19) {
+          task.jvmArgs += ["--enable-preview"]
+        }
       }
     }
 

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.3'
+  api 'de.thetaphi:forbiddenapis:3.4'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.6.2'

buildSrc/src/main/java/org/opensearch/gradle/Jdk.java

@@ -37,6 +37,7 @@
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.Property;
 import org.gradle.api.tasks.TaskDependency;
+import org.gradle.internal.os.OperatingSystem;
 
 import java.io.File;
 import java.util.Arrays;
@@ -169,7 +170,7 @@ public Object getBinJavaPath() {
         return new Object() {
             @Override
             public String toString() {
-                return getHomeRoot() + "/bin/java";
+                return OperatingSystem.current().getExecutableName(getHomeRoot() + "/bin/java");
             }
         };
     }

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -111,6 +111,9 @@ public void execute(Task t) {
                         test.systemProperty("java.locale.providers", "SPI,COMPAT");
                         test.jvmArgs("--illegal-access=warn");
                     }
+                    if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) {
+                        test.jvmArgs("-Djava.security.manager=allow");
+                    }
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -45,12 +45,15 @@
 import org.gradle.internal.jvm.Jvm;
 import org.gradle.internal.jvm.inspection.JvmInstallationMetadata;
 import org.gradle.internal.jvm.inspection.JvmMetadataDetector;
+import org.gradle.jvm.toolchain.internal.InstallationLocation;
 import org.gradle.util.GradleVersion;
 
 import javax.inject.Inject;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -195,7 +198,29 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java
     }
 
     private JvmInstallationMetadata getJavaInstallation(File javaHome) {
-        return jvmMetadataDetector.getMetadata(javaHome);
+        final InstallationLocation location = new InstallationLocation(javaHome, "Java home");
+
+        try {
+            try {
+                // The getMetadata(File) is used by Gradle pre-7.6
+                return (JvmInstallationMetadata) MethodHandles.publicLookup()
+                    .findVirtual(JvmMetadataDetector.class, "getMetadata", MethodType.methodType(JvmInstallationMetadata.class, File.class))
+                    .bindTo(jvmMetadataDetector)
+                    .invokeExact(location.getLocation());
+            } catch (NoSuchMethodException | IllegalAccessException ex) {
+                // The getMetadata(InstallationLocation) is used by Gradle post-7.6
+                return (JvmInstallationMetadata) MethodHandles.publicLookup()
+                    .findVirtual(
+                        JvmMetadataDetector.class,
+                        "getMetadata",
+                        MethodType.methodType(JvmInstallationMetadata.class, InstallationLocation.class)
+                    )
+                    .bindTo(jvmMetadataDetector)
+                    .invokeExact(location);
+            }
+        } catch (Throwable ex) {
+            throw new IllegalStateException("Unable to find suitable JvmMetadataDetector::getMetadata", ex);
+        }
     }
 
     private List<JavaHome> getAvailableJavaVersions(JavaVersion minimumCompilerVersion) {
@@ -205,7 +230,7 @@ private List<JavaHome> getAvailableJavaVersions(JavaVersion minimumCompilerVersi
             String javaHomeEnvVarName = getJavaHomeEnvVarName(Integer.toString(version));
             if (System.getenv(javaHomeEnvVarName) != null) {
                 File javaHomeDirectory = new File(findJavaHome(Integer.toString(version)));
-                JvmInstallationMetadata javaInstallation = jvmMetadataDetector.getMetadata(javaHomeDirectory);
+                JvmInstallationMetadata javaInstallation = getJavaInstallation(javaHomeDirectory);
                 JavaHome javaHome = JavaHome.of(version, providers.provider(() -> {
                     int actualVersion = Integer.parseInt(javaInstallation.getLanguageVersion().getMajorVersion());
                     if (actualVersion != version) {

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         project.getConfigurations().create("forbiddenApisCliJar");
-        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.2");
+        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.4");
 
         Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
         if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) {

buildSrc/version.properties

@@ -2,8 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.5.0-snapshot-a4ef70f
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 17.0.5+8
-
+bundled_jdk = 19.0.1+10
 
 
 # optional dependencies

client/rest/src/main/java/org/opensearch/client/Response.java

@@ -147,7 +147,7 @@ public HttpEntity getEntity() {
      * @return {@code true} if the input string matches the specification
      */
     private static boolean matchWarningHeaderPatternByPrefix(final String s) {
-        return s.startsWith("299 OpenSearch-") || s.startsWith("299 Elasticsearch-");
+        return s.startsWith("299 OpenSearch-");
     }
 
     /**

distribution/src/config/jvm.options

@@ -78,3 +78,6 @@ ${error.file}
 
 # Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380)
 18-:-Djava.security.manager=allow
+
+# Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4 (https://github.com/opensearch-project/OpenSearch/issues/4637)
+19-:--enable-preview

distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java

@@ -77,12 +77,21 @@ static List<String> systemJvmOptions() {
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
-
+                // security manager
+                allowSecurityManagerOption(),
                 javaLocaleProviders()
             )
         ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
+    private static String allowSecurityManagerOption() {
+        if (Runtime.version().feature() > 17) {
+            return "-Djava.security.manager=allow";
+        } else {
+            return "";
+        }
+    }
+
     private static String maybeShowCodeDetailsInExceptionMessages() {
         if (Runtime.version().feature() >= 14) {
             return "-XX:+ShowCodeDetailsInExceptionMessages";

distribution/tools/plugin-cli/build.gradle

@@ -35,7 +35,7 @@ archivesBaseName = 'opensearch-plugin-cli'
 dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
-  api "org.bouncycastle:bcpg-fips:1.0.5.1"
+  api "org.bouncycastle:bcpg-fips:1.0.7.1"
   api "org.bouncycastle:bc-fips:1.0.2.3"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.2'

distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.7.1.jar.sha1

@@ -0,0 +1 @@
+5e1952428655ea822066f86df2e3ecda8fa0ba2b

gradle/runtime-jdk-provision.gradle

@@ -20,7 +20,11 @@ if (BuildParams.getIsRuntimeJavaHomeSet()) {
   configure(allprojects - project(':build-tools')) {
     project.tasks.withType(Test).configureEach { Test test ->
       if (BuildParams.getIsRuntimeJavaHomeSet()) {
-        test.executable = "${BuildParams.runtimeJavaHome}/bin/java"
+        if (OS.current() == OS.WINDOWS) {
+          test.executable = "${BuildParams.runtimeJavaHome}/bin/java.exe"
+        } else {
+          test.executable = "${BuildParams.runtimeJavaHome}/bin/java"
+        }
       }
     }
   }

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-rc-3-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219
+distributionSha256Sum=abc6de2653ec9befb00cc0d064ce1ca4e4dab2c91955e830661505189c0b2f08

modules/ingest-geoip/build.gradle

@@ -39,7 +39,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:3.0.1')
+  api('com.maxmind.geoip2:geoip2:3.0.2')
   // geoip2 dependencies:
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")

modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1

@@ -0,0 +1 @@
+f0ab0a451309c93f0fb6bf3cb203ba19d452c800

modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java

@@ -546,7 +546,6 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
         String name = this.name != null ? this.name : pft.name();
         QueryShardContext percolateShardContext = wrap(context);
         PercolatorFieldMapper.configureContext(percolateShardContext, pft.mapUnmappedFieldsAsText);
-        ;
         PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField, percolateShardContext);
 
         return pft.percolateQuery(name, queryStore, documents, docSearcher, excludeNestedDocuments, context.indexVersionCreated());

plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java

@@ -79,6 +79,9 @@ public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, St
 
     private static String parse(String rule, Set<String> dup) {
         String[] values = CSVUtil.parse(rule);
+        if (values.length == 0) {
+            throw new IllegalArgumentException("Malformed csv in user dictionary.");
+        }
         if (dup.add(values[0]) == false) {
             throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary.");
         }

plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java

@@ -379,6 +379,15 @@ public void testKuromojiAnalyzerInvalidUserDictOption() throws Exception {
         );
     }
 
+    public void testKuromojiAnalyzerEmptyDictRule() throws Exception {
+        Settings settings = Settings.builder()
+            .put("index.analysis.analyzer.my_analyzer.type", "kuromoji")
+            .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "\"")
+            .build();
+        RuntimeException exc = expectThrows(RuntimeException.class, () -> createTestAnalysis(settings));
+        assertThat(exc.getMessage(), equalTo("Line [1]: Malformed csv in user dictionary."));
+    }
+
     public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception {
         Settings settings = Settings.builder()
             .put("index.analysis.analyzer.my_analyzer.type", "kuromoji")

plugins/ingest-attachment/build.gradle

@@ -84,7 +84,7 @@ dependencies {
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"
   // Apple iWork
-  api 'org.apache.commons:commons-compress:1.21'
+  api 'org.apache.commons:commons-compress:1.22'
   // Outlook documents
   api "org.apache.james:apache-mime4j-core:${versions.mime4j}"
   api "org.apache.james:apache-mime4j-dom:${versions.mime4j}"

plugins/ingest-attachment/licenses/commons-compress-1.22.jar.sha1

@@ -0,0 +1 @@
+691a8b4e6cf4248c3bc72c8b719337d5cb7359fa

plugins/repository-azure/build.gradle

@@ -54,7 +54,7 @@ dependencies {
   api "io.netty:netty-resolver-dns:${versions.netty}"
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
-  api 'com.azure:azure-storage-blob:12.16.1'
+  api 'com.azure:azure-storage-blob:12.20.0'
   api 'org.reactivestreams:reactive-streams:1.0.4'
   api 'io.projectreactor:reactor-core:3.4.23'
   api 'io.projectreactor.netty:reactor-netty:1.0.18'

plugins/repository-azure/licenses/azure-storage-blob-12.20.0.jar.sha1

@@ -0,0 +1 @@
+e682920b0e3115433f25d65b0718f8763035357e

plugins/repository-gcs/build.gradle

@@ -81,7 +81,7 @@ dependencies {
   api 'com.google.api:gax-httpjson:0.103.1'
   api 'io.grpc:grpc-context:1.46.0'
   api 'io.opencensus:opencensus-api:0.18.0'
-  api 'io.opencensus:opencensus-contrib-http-util:0.18.0'
+  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
   api 'com.google.apis:google-api-services-storage:v1-rev20220608-1.32.1'
 
   testImplementation project(':test:fixtures:gcs-fixture')
@@ -209,6 +209,8 @@ thirdPartyAudit {
     'javax.jms.Message',
     'javax.servlet.ServletContextEvent',
     'javax.servlet.ServletContextListener',
+    'io.opencensus.tags.TagMetadata',
+    'io.opencensus.tags.TagMetadata$TagTtl',
   )
 }
 

plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1

@@ -0,0 +1 @@
+3c13fc5715231fadb16a9b74a44d9d59c460cfa8

plugins/repository-hdfs/build.gradle

@@ -67,7 +67,7 @@ dependencies {
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   api 'org.apache.avro:avro:1.11.1'
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api 'com.google.code.gson:gson:2.9.0'
+  api 'com.google.code.gson:gson:2.10'
   runtimeOnly 'com.google.guava:guava:31.1-jre'
   api 'com.google.protobuf:protobuf-java:3.21.7'
   api "commons-logging:commons-logging:${versions.commonslogging}"

plugins/repository-hdfs/licenses/gson-2.10.jar.sha1

@@ -0,0 +1 @@
+dd9b193aef96e973d5a11ab13cd17430c2e4306b