merge changes from main

Signed-off-by: suranjay <surajkumar.tu@gmail.com>

.github/workflows/gradle-check.yml

@@ -22,52 +22,52 @@ jobs:
     timeout-minutes: 130
     steps:
       - name: Checkout OpenSearch repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Setup environment variables (PR)
         if: github.event_name == 'pull_request_target'
         run: |
-            echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
-            echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+          echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
 
       - name: Setup environment variables (Push)
         if: github.event_name == 'push'
         run: |
-            repo_url="https://github.com/opensearch-project/OpenSearch"
-            ref_id=$(git rev-parse HEAD)
-            branch_name=$(git rev-parse --abbrev-ref HEAD)
-            echo "branch_name=$branch_name" >> $GITHUB_ENV
-            echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
-            echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
-            echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
-            echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
-            echo "pr_number=Null" >> $GITHUB_ENV
+          repo_url="https://github.com/opensearch-project/OpenSearch"
+          ref_id=$(git rev-parse HEAD)
+          branch_name=$(git rev-parse --abbrev-ref HEAD)
+          echo "branch_name=$branch_name" >> $GITHUB_ENV
+          echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
+          echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
+          echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
+          echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
+          echo "pr_number=Null" >> $GITHUB_ENV
 
       - name: Checkout opensearch-build repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: opensearch-project/opensearch-build
           ref: main
           path: opensearch-build
 
       - name: Trigger jenkins workflow to run gradle check
         run: |
-            set -e
-            set -o pipefail
-            bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
+          set -e
+          set -o pipefail
+          bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
 
       - name: Setup Result Status
         if: always()
         run: |
-            WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'`
-            RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'`
-            echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV
-            echo "result=$RESULT" >> $GITHUB_ENV
+          WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'`
+          RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'`
+          echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV
+          echo "result=$RESULT" >> $GITHUB_ENV
 
       - name: Upload Coverage Report
         if: success()
@@ -81,25 +81,25 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-              ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :white_check_mark:
-              * **URL:** ${{ env.workflow_url }}
-              * **CommitID:** ${{ env.pr_from_sha }}
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :white_check_mark:
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
 
       - name: Extract Test Failure
         if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
         run: |
-            TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
-            if [[ "$TEST_FAILURES" != "" ]]
-            then
-              echo "test_failures<<EOF" >> $GITHUB_ENV
-              echo "" >> $GITHUB_ENV
-              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
-              echo '```' >> $GITHUB_ENV
-              echo "$TEST_FAILURES" >> $GITHUB_ENV
-              echo '```' >> $GITHUB_ENV
-              echo "EOF" >> $GITHUB_ENV
-            fi
+          TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
+          if [[ "$TEST_FAILURES" != "" ]]
+          then
+            echo "test_failures<<EOF" >> $GITHUB_ENV
+            echo "" >> $GITHUB_ENV
+            echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+            echo '```' >> $GITHUB_ENV
+            echo "$TEST_FAILURES" >> $GITHUB_ENV
+            echo '```' >> $GITHUB_ENV
+            echo "EOF" >> $GITHUB_ENV
+          fi
 
       - name: Create Comment Flaky
         if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
@@ -119,12 +119,12 @@ jobs:
         with:
           issue-number: ${{ env.pr_number }}
           body: |
-              ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
-              * **URL:** ${{ env.workflow_url }}
-              * **CommitID:** ${{ env.pr_from_sha }}
-              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
-              Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
+            Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
+            Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
 
       - name: Create Issue On Push Failure
         if: ${{ github.event_name == 'push' && failure() }}

.github/workflows/precommit.yml

@@ -1,12 +1,12 @@
 name: Gradle Precommit
 on: [pull_request]
- 
+
 jobs:
   precommit:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest] 
+        os: [windows-latest, macos-latest]  # precommit on ubuntu-latest is run as a part of the gradle-check workflow
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 11

CHANGELOG.md

@@ -97,13 +97,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901))
 - Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 - Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
-- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.83 ([7968](https://github.com/opensearch-project/OpenSearch/pull/7968))
+- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.85 ([7968], #8255)
 - Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068))
 - Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre (#8009)
 - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 (#8014, #8013, #8010)
 - Bump `com.diffplug.spotless` from 6.18.0 to 6.19.0 (#8007)
 - Bump `'com.azure:azure-storage-blob` to 12.22.2 from 12.21.1 ([#8043](https://github.com/opensearch-project/OpenSearch/pull/8043))
-- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015)
+- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015, #8254)
 - Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre ([#8011](https://github.com/opensearch-project/OpenSearch/pull/8011), [#8012](https://github.com/opensearch-project/OpenSearch/pull/8012), [#8107](https://github.com/opensearch-project/OpenSearch/pull/8107))
 - Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016))
 - Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122))
@@ -112,6 +112,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140))
 - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181))
 - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191))
+- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257)
+- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256)
+- [Upgrade] Lucene 9.7.0 release (#8272)
+- Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
@@ -131,6 +135,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Removed
 - Remove `COMPRESSOR` variable from `CompressorFactory` and use `DEFLATE_COMPRESSOR` instead ([7907](https://github.com/opensearch-project/OpenSearch/pull/7907))
+- Remove concurrency based minimum file cache size restriction ([#8294](https://github.com/opensearch-project/OpenSearch/pull/8294))
 
 ### Fixed
 - Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924

buildSrc/build.gradle

@@ -118,9 +118,9 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.2'
-  api 'com.networknt:json-schema-validator:1.0.84'
+  api 'com.networknt:json-schema-validator:1.0.85'
   api 'org.jruby.jcodings:jcodings:1.0.58'
-  api 'org.jruby.joni:joni:2.1.48'
+  api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
   api "org.ajoberstar.grgit:grgit-core:5.2.0"
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.7.0-snapshot-204acc3
+lucene            = 9.7.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.1+9
@@ -65,6 +65,8 @@ zstd              = 1.5.5-3
 
 jzlib             = 1.1.3
 
+resteasy          = 6.2.4.Final
+
 # opentelemetry dependencies
 opentelemetry    = 1.26.0
 

libs/core/licenses/lucene-core-9.7.0.jar.sha1

@@ -0,0 +1 @@
+ad391210ffd806931334be9670a35af00c56f959

libs/core/src/main/java/org/opensearch/Version.java

@@ -89,7 +89,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0);
     public static final Version V_2_8_1 = new Version(2080199, org.apache.lucene.util.Version.LUCENE_9_6_0);
-    public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_6_0);
+    public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version CURRENT = V_3_0_0;
 

modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1

@@ -0,0 +1 @@
+297e1cfade4ef71466cc9d4f361d81807c8dc4c8

plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1

@@ -0,0 +1 @@
+94293b169fb8572f440a5a4a523320ecf9778ffe

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1

@@ -0,0 +1 @@
+2df800a38b64867b8dcd61fc2cd986114e4a80cb

plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1

@@ -0,0 +1 @@
+a01e8153f34d72e8c8c0180c1dea5b10f677dd3a

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1

@@ -0,0 +1 @@
+b7d47d54683b0b1e09b271c32d1b7d3eb1990f49

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1

@@ -0,0 +1 @@
+5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1

@@ -0,0 +1 @@
+d23b1f05b471e05d0d6068b3ece7c8c65672eae7

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1

@@ -0,0 +1 @@
+dfb4313f3c68d337310522840d7144c1605d084a

plugins/repository-azure/build.gradle

@@ -60,7 +60,7 @@ dependencies {
   api 'io.projectreactor:reactor-core:3.5.6'
   api 'io.projectreactor.netty:reactor-netty:1.1.7'
   api 'io.projectreactor.netty:reactor-netty-core:1.1.7'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.7'
+  api 'io.projectreactor.netty:reactor-netty-http:1.1.8'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"

plugins/repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1

@@ -0,0 +1 @@
+696ea25658295e49906c6aad13fa70acbdeb2359

qa/mixed-cluster/build.gradle

@@ -55,6 +55,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
   }
 
   String baseName = "v${bwcVersion}"
+  String bwcVersionStr = "${bwcVersion}"
 
   /* This project runs the core REST tests against a 4 node cluster where two of
      the nodes has a different minor.  */
@@ -86,6 +87,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
       nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
     }
+    systemProperty 'tests.upgrade_from_version', bwcVersionStr
     systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
     onlyIf { project.bwc_tests_enabled }
   }

qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java

@@ -66,6 +66,9 @@
 
 public class IndexingIT extends OpenSearchRestTestCase {
 
+    protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
+
+
     private int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
         for (int i = 0; i < numDocs; i++) {
             final int id = idStart + i;
@@ -114,12 +117,16 @@ private void printClusterRouting() throws IOException, ParseException {
      * @throws Exception
      */
     public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
+        if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) {
+            logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION);
+            return;
+        }
         Nodes nodes = buildNodeAndVersions();
         assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
         logger.info("cluster discovered:\n {}", nodes.toString());
         final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.toList());
         final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
-        // Exclude bwc nodes from allocation so that primaries gets allocated on current version
+        // Update allocation settings so that primaries gets allocated only on nodes running on older version
         Settings.Builder settings = Settings.builder()
             .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
             .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
@@ -133,7 +140,7 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
         try (RestClient nodeClient = buildClient(restClientSettings(),
             nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) {
 
-            logger.info("allowing replica shards assignment on bwc nodes");
+            logger.info("Remove allocation include settings so that shards can be allocated on current version nodes");
             updateIndexSettings(index, Settings.builder().putNull("index.routing.allocation.include._name"));
             // Add replicas so that it can be assigned on higher OS version nodes.
             updateIndexSettings(index, Settings.builder().put("index.number_of_replicas", 2));
@@ -154,13 +161,17 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
 
 
     /**
-     * This test creates a cluster with primary on older version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider};
+     * This test creates a cluster with primary on higher version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider};
      * replica shard allocation on lower OpenSearch version is prevented. Thus, this test though cover the use case where
      * primary shard containing nodes are running on higher OS version while replicas are unassigned.
      *
      * @throws Exception
      */
     public void testIndexingWithReplicaOnBwcNodes() throws Exception {
+        if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) {
+            logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION);
+            return;
+        }
         Nodes nodes = buildNodeAndVersions();
         assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
         logger.info("cluster discovered:\n {}", nodes.toString());