Merge branch 'main' into primary-check

CHANGELOG.md

@@ -20,8 +20,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add BWC version 2.3.1 ([#4513](https://github.com/opensearch-project/OpenSearch/pull/4513))
 - [Segment Replication] Add snapshot and restore tests for segment replication feature ([#3993](https://github.com/opensearch-project/OpenSearch/pull/3993))
 - Added missing javadocs for `:example-plugins` modules ([#4540](https://github.com/opensearch-project/OpenSearch/pull/4540))
+- Add missing Javadoc tag descriptions in unit tests ([#4629](https://github.com/opensearch-project/OpenSearch/pull/4629))
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
+- Bumps `reactor-netty-http` from 1.0.18 to 1.0.23
+- Bumps `jettison` from 1.5.0 to 1.5.1
 
 
 ### Dependencies
@@ -35,6 +38,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
+- Bumps `hadoop-hdfs` from 3.3.3 to 3.3.4 ([#4644](https://github.com/opensearch-project/OpenSearch/pull/4644))
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
@@ -49,11 +53,14 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Unmute test RelocationIT.testRelocationWhileIndexingRandom ([#4580](https://github.com/opensearch-project/OpenSearch/pull/4580))
 - Add DecommissionService and helper to execute awareness attribute decommissioning ([#4084](https://github.com/opensearch-project/OpenSearch/pull/4084))
 - Further simplification of the ZIP publication implementation ([#4360](https://github.com/opensearch-project/OpenSearch/pull/4360))
+- Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
+- Load the deprecated master role in a dedicated method instead of in setAdditionalRoles() ([#4582](https://github.com/opensearch-project/OpenSearch/pull/4582))
 
 ### Deprecated
 
 ### Removed
 - Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568))
+- Unused object and import within TransportClusterAllocationExplainAction ([#4639](https://github.com/opensearch-project/OpenSearch/pull/4639))
 
 ### Fixed
 - `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289))
@@ -73,14 +80,16 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Segment Replication] Update flaky testOnNewCheckpointFromNewPrimaryCancelOngoingReplication unit test ([#4414](https://github.com/opensearch-project/OpenSearch/pull/4414))
 - Fixed the `_cat/shards/10_basic.yml` test cases fix.
 - [Segment Replication] Fix timeout issue by calculating time needed to process getSegmentFiles ([#4426](https://github.com/opensearch-project/OpenSearch/pull/4426))
-- [Bug]: gradle check failing with java heap OutOfMemoryError (([#4328](https://github.com/opensearch-project/OpenSearch/
+- [Bug]: gradle check failing with java heap OutOfMemoryError ([#4328](https://github.com/opensearch-project/OpenSearch/))
 - `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
 - Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469))
 - Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
 - Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
 - [Segment Replication] Ignore lock file when testing cleanupAndPreserveLatestCommitPoint ([#4544](https://github.com/opensearch-project/OpenSearch/pull/4544))
 - Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
 - [Segment Replication] Adding check to make sure checkpoint is not processed when a shard's shard routing is primary ([#4630](https://github.com/opensearch-project/OpenSearch/pull/4630))
+- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613))
+- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

buildSrc/src/main/java/org/opensearch/gradle/Jdk.java

@@ -128,7 +128,7 @@ public void setArchitecture(final String architecture) {
                 "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES
             );
         }
-        this.architecture.set(architecture);
+        this.architecture.set(jdkArchitecture);
     }
 
     public String getBaseVersion() {

modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsGeoShapeAggregatorTests.java

@@ -68,7 +68,7 @@ protected List<SearchPlugin> getSearchPlugins() {
     /**
      * Testing Empty aggregator results.
      *
-     * @throws Exception
+     * @throws Exception if an error occurs accessing the index
      */
     public void testEmpty() throws Exception {
         try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
@@ -93,7 +93,7 @@ public void testEmpty() throws Exception {
     /**
      * Testing GeoBoundAggregator for random shapes which are indexed.
      *
-     * @throws Exception
+     * @throws Exception if an error occurs accessing the index
      */
     public void testRandom() throws Exception {
         final int numDocs = randomIntBetween(50, 100);

modules/transport-netty4/src/main/java/org/opensearch/http/netty4/Netty4HttpServerTransport.java

@@ -332,8 +332,10 @@ public ChannelHandler configureServerChannelHandler() {
         return new HttpChannelHandler(this, handlingSettings);
     }
 
-    static final AttributeKey<Netty4HttpChannel> HTTP_CHANNEL_KEY = AttributeKey.newInstance("opensearch-http-channel");
-    static final AttributeKey<Netty4HttpServerChannel> HTTP_SERVER_CHANNEL_KEY = AttributeKey.newInstance("opensearch-http-server-channel");
+    protected static final AttributeKey<Netty4HttpChannel> HTTP_CHANNEL_KEY = AttributeKey.newInstance("opensearch-http-channel");
+    protected static final AttributeKey<Netty4HttpServerChannel> HTTP_SERVER_CHANNEL_KEY = AttributeKey.newInstance(
+        "opensearch-http-server-channel"
+    );
 
     protected static class HttpChannelHandler extends ChannelInitializer<Channel> {
 

plugins/discovery-azure-classic/build.gradle

@@ -59,7 +59,7 @@ dependencies {
   api "com.sun.jersey:jersey-client:${versions.jersey}"
   api "com.sun.jersey:jersey-core:${versions.jersey}"
   api "com.sun.jersey:jersey-json:${versions.jersey}"
-  api 'org.codehaus.jettison:jettison:1.5.0'
+  api 'org.codehaus.jettison:jettison:1.5.1'
   api 'com.sun.xml.bind:jaxb-impl:2.2.3-1'
 
   // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,

plugins/discovery-azure-classic/licenses/jettison-1.5.1.jar.sha1

@@ -0,0 +1 @@
+d8918f348f234f5046bd39ea1ed9fc91deac402f

plugins/repository-azure/build.gradle

@@ -59,7 +59,7 @@ dependencies {
   api 'io.projectreactor:reactor-core:3.4.23'
   api 'io.projectreactor.netty:reactor-netty:1.0.18'
   api 'io.projectreactor.netty:reactor-netty-core:1.0.22'
-  api 'io.projectreactor.netty:reactor-netty-http:1.0.18'
+  api 'io.projectreactor.netty:reactor-netty-http:1.0.23'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"

plugins/repository-azure/licenses/reactor-netty-http-1.0.23.jar.sha1

@@ -0,0 +1 @@
+63932f2b675f451135986b3723a12d45e818b170

plugins/repository-hdfs/build.gradle

@@ -48,7 +48,7 @@ opensearchplugin {
 }
 
 versions << [
-  'hadoop3': '3.3.3'
+  'hadoop3': '3.3.4'
 ]
 
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"

plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1

@@ -0,0 +1 @@
+6339a8f7279310c8b1f7ef314b592d8c71ca72ef

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1

@@ -0,0 +1 @@
+21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0

plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1

@@ -0,0 +1 @@
+036ef2f86dc44410d2bb5d54ce40435d2484d9a5

server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java

@@ -50,7 +50,6 @@
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation.DebugMode;
 import org.opensearch.cluster.routing.allocation.ShardAllocationDecision;
-import org.opensearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
@@ -77,7 +76,6 @@ public class TransportClusterAllocationExplainAction extends TransportClusterMan
     private final ClusterInfoService clusterInfoService;
     private final SnapshotsInfoService snapshotsInfoService;
     private final AllocationDeciders allocationDeciders;
-    private final ShardsAllocator shardAllocator;
     private final AllocationService allocationService;
 
     @Inject
@@ -90,7 +88,6 @@ public TransportClusterAllocationExplainAction(
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService,
         AllocationDeciders allocationDeciders,
-        ShardsAllocator shardAllocator,
         AllocationService allocationService
     ) {
         super(
@@ -105,7 +102,6 @@ public TransportClusterAllocationExplainAction(
         this.clusterInfoService = clusterInfoService;
         this.snapshotsInfoService = snapshotsInfoService;
         this.allocationDeciders = allocationDeciders;
-        this.shardAllocator = shardAllocator;
         this.allocationService = allocationService;
     }
 

server/src/main/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -188,7 +188,7 @@ private RolloverResult rolloverAlias(
         ClusterState newState = createIndexService.applyCreateIndexRequest(currentState, createIndexClusterStateRequest, silent);
         newState = indexAliasesService.applyAliasActions(
             newState,
-            rolloverAliasToNewIndex(sourceIndexName, rolloverIndexName, explicitWriteIndex, aliasMetadata.isHidden(), aliasName)
+            rolloverAliasToNewIndex(sourceIndexName, rolloverIndexName, explicitWriteIndex, aliasMetadata, aliasName)
         );
 
         RolloverInfo rolloverInfo = new RolloverInfo(aliasName, metConditions, threadPool.absoluteTimeInMillis());
@@ -309,20 +309,46 @@ static List<AliasAction> rolloverAliasToNewIndex(
         String oldIndex,
         String newIndex,
         boolean explicitWriteIndex,
-        @Nullable Boolean isHidden,
+        AliasMetadata aliasMetadata,
         String alias
     ) {
+        String filterAsString = aliasMetadata.getFilter() != null ? aliasMetadata.getFilter().string() : null;
+
         if (explicitWriteIndex) {
             return Collections.unmodifiableList(
                 Arrays.asList(
-                    new AliasAction.Add(newIndex, alias, null, null, null, true, isHidden),
-                    new AliasAction.Add(oldIndex, alias, null, null, null, false, isHidden)
+                    new AliasAction.Add(
+                        newIndex,
+                        alias,
+                        filterAsString,
+                        aliasMetadata.getIndexRouting(),
+                        aliasMetadata.getSearchRouting(),
+                        true,
+                        aliasMetadata.isHidden()
+                    ),
+                    new AliasAction.Add(
+                        oldIndex,
+                        alias,
+                        filterAsString,
+                        aliasMetadata.getIndexRouting(),
+                        aliasMetadata.getSearchRouting(),
+                        false,
+                        aliasMetadata.isHidden()
+                    )
                 )
             );
         } else {
             return Collections.unmodifiableList(
                 Arrays.asList(
-                    new AliasAction.Add(newIndex, alias, null, null, null, null, isHidden),
+                    new AliasAction.Add(
+                        newIndex,
+                        alias,
+                        filterAsString,
+                        aliasMetadata.getIndexRouting(),
+                        aliasMetadata.getSearchRouting(),
+                        null,
+                        aliasMetadata.isHidden()
+                    ),
                     new AliasAction.Remove(oldIndex, alias, null)
                 )
             );

server/src/main/java/org/opensearch/cluster/metadata/AliasAction.java

@@ -138,6 +138,18 @@ public String getAlias() {
             return alias;
         }
 
+        public String getFilter() {
+            return filter;
+        }
+
+        public String getSearchRouting() {
+            return searchRouting;
+        }
+
+        public String getIndexRouting() {
+            return indexRouting;
+        }
+
         public Boolean writeIndex() {
             return writeIndex;
         }

server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java

@@ -616,11 +616,18 @@ public static void setAdditionalRoles(final Set<DiscoveryNodeRole> additionalRol
             + "], roles by name abbreviation ["
             + roleNameAbbreviationToPossibleRoles
             + "]";
-        // TODO: Remove the Map 'roleNameToPossibleRolesWithMaster' and let 'roleMap = roleNameToPossibleRoles', after removing MASTER_ROLE.
-        // It's used to allow CLUSTER_MANAGER_ROLE that introduced in 2.0, having the same abbreviation name with MASTER_ROLE.
-        final Map<String, DiscoveryNodeRole> roleNameToPossibleRolesWithMaster = new HashMap<>(roleNameToPossibleRoles);
-        roleNameToPossibleRolesWithMaster.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), DiscoveryNodeRole.MASTER_ROLE);
-        roleMap = Collections.unmodifiableMap(roleNameToPossibleRolesWithMaster);
+        roleMap = roleNameToPossibleRoles;
+    }
+
+    /**
+     * Load the deprecated {@link DiscoveryNodeRole#MASTER_ROLE}.
+     * Master role is not added into BUILT_IN_ROLES, because {@link #setAdditionalRoles(Set)} check role name abbreviation duplication,
+     * and CLUSTER_MANAGER_ROLE has the same abbreviation name with MASTER_ROLE.
+     */
+    public static void setDeprecatedMasterRole() {
+        final Map<String, DiscoveryNodeRole> modifiableRoleMap = new HashMap<>(roleMap);
+        modifiableRoleMap.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), DiscoveryNodeRole.MASTER_ROLE);
+        roleMap = Collections.unmodifiableMap(modifiableRoleMap);
     }
 
     public static Set<String> getPossibleRoleNames() {

server/src/main/java/org/opensearch/node/Node.java

@@ -429,6 +429,8 @@ protected Node(
                 .collect(Collectors.toSet());
             DiscoveryNode.setAdditionalRoles(additionalRoles);
 
+            DiscoveryNode.setDeprecatedMasterRole();
+
             /*
              * Create the environment based on the finalized view of the settings. This is to ensure that components get the same setting
              * values, no matter they ask for them from.