Merge branch 'main' into main

opensearch-project · Sep 1, 2022 · 0adabb1 · 0adabb1
2 parents 48a4a4e + 715da84
commit 0adabb1
Show file tree

Hide file tree

Showing 56 changed files with 72 additions and 34 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,11 +10,15 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added release notes for 2.2.1 ([#4344](https://github.com/opensearch-project/OpenSearch/pull/4344))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
+### Dependencies
+- Bumps `com.diffplug.spotless` from 6.9.1 to 6.10.0
+- Bumps `xmlbeans` from 5.1.0 to 5.1.1
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
+- Update to Netty 4.1.80.Final ([#4359](https://github.com/opensearch-project/OpenSearch/pull/4359))
 
 ### Deprecated
 
@@ -29,7 +33,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add timeout on Mockito.verify to reduce flakyness in testReplicationOnDone test([#4314](https://github.com/opensearch-project/OpenSearch/pull/4314))
 - Commit workflow for dependabot changelog helper ([#4331](https://github.com/opensearch-project/OpenSearch/pull/4331))
 - Fixed cancellation of segment replication events ([#4225](https://github.com/opensearch-project/OpenSearch/pull/4225))
-- Bugs for dependabot changelog verifier workflow ([#4364](https://github.com/opensearch-project/OpenSearch/pull/4364)) 
+- Bugs for dependabot changelog verifier workflow ([#4364](https://github.com/opensearch-project/OpenSearch/pull/4364))
+- Fix flaky random test `NRTReplicationEngineTests.testUpdateSegments` ([#4352](https://github.com/opensearch-project/OpenSearch/pull/4352))
 - `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
 
 ### Security
@@ -54,4 +59,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 
 [Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD
-[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x
+[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x
diff --git a/build.gradle b/build.gradle
@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.9.1" apply false
+  id "com.diffplug.spotless" version "6.10.0" apply false
   id "org.gradle.test-retry" version "1.4.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -21,7 +21,7 @@ asm               = 9.3
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.79.Final
+netty             = 4.1.80.Final
 joda              = 2.10.13
 
 # client dependencies

diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
@@ -144,6 +144,14 @@ thirdPartyAudit {
     'org.apache.log4j.Level',
     'org.apache.log4j.Logger',
 
+    // from io.netty.handler.ssl.OpenSslEngine (netty)
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
+
     // from io.netty.handler.ssl.OpenSslEngine (netty)
     'io.netty.internal.tcnative.Buffer',
     'io.netty.internal.tcnative.CertificateCompressionAlgo',

diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+a087321a63d9991e25f7b7d24ef53edcbcb954ff
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+4941821a158d16311665d8606aefa610ecf0f64c
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+efb23f9d5187d2f733595ef7930137f0cb2cec48
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+bf7b66834188ef1a6f6095291c6b81a1880798ba
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+3d43ce22863bc590e4e33fbdabbb58dc05f4c43d
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+cf7029d2f9bc4eeae8ff15af7a528d06b518a017
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+3bbb0d4bfbbab867e5b757b97a6e5e0d1348d94c
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+57fcace7a1b8567aa39921c915d1b1ba78fd4d2d
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.79.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.79.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.80.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+da3d7da1a8d317ae2c82b400fd255fe610c43ebe
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
@@ -79,7 +79,7 @@ dependencies {
   api "org.apache.poi:poi:${versions.poi}"
   api "org.apache.poi:poi-ooxml-lite:${versions.poi}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'org.apache.xmlbeans:xmlbeans:5.1.0'
+  api 'org.apache.xmlbeans:xmlbeans:5.1.1'
   api 'org.apache.commons:commons-collections4:4.4'
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"

diff --git a/plugins/ingest-attachment/licenses/xmlbeans-5.1.0.jar.sha1 b/plugins/ingest-attachment/licenses/xmlbeans-5.1.0.jar.sha1
diff --git a/plugins/ingest-attachment/licenses/xmlbeans-5.1.1.jar.sha1 b/plugins/ingest-attachment/licenses/xmlbeans-5.1.1.jar.sha1
@@ -0,0 +1 @@
+48a369df0eccb509d46203104e4df9cb00f0f68b
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.79.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.80.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+6926d2ea779f41071ecb1948d880dfbb3a6ee126
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.79.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.80.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+bf7b66834188ef1a6f6095291c6b81a1880798ba
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.79.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.80.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+00025b767be3425f3b31a34ee095c85619169f17
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.79.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.80.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+9b3b42ff805723fb98120f5ab2019c53e71da91b
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.79.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.80.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+6b1602f80b6235b0b7d53bc5e9c1a6cd11c1b804
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.79.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.80.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+da3d7da1a8d317ae2c82b400fd255fe610c43ebe
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.79.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.79.Final.jar.sha1
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.80.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+39e73b76a3ec65df731b371179e15f2c3e4e7575
diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle
@@ -83,6 +83,12 @@ thirdPartyAudit {
     'org.bouncycastle.cert.X509v3CertificateBuilder',
     'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
     'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
 
     // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
     'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',

diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+a087321a63d9991e25f7b7d24ef53edcbcb954ff
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+4941821a158d16311665d8606aefa610ecf0f64c
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+efb23f9d5187d2f733595ef7930137f0cb2cec48
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+3d43ce22863bc590e4e33fbdabbb58dc05f4c43d
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+cf7029d2f9bc4eeae8ff15af7a528d06b518a017
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+3bbb0d4bfbbab867e5b757b97a6e5e0d1348d94c
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.79.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.79.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.80.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.80.Final.jar.sha1
@@ -0,0 +1 @@
+57fcace7a1b8567aa39921c915d1b1ba78fd4d2d
diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java
@@ -112,10 +112,14 @@ public void testUpdateSegments() throws Exception {
             final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore)
         ) {
             // add docs to the primary engine.
-            List<Engine.Operation> operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean())
-                .stream()
-                .filter(op -> op.operationType().equals(Engine.Operation.TYPE.INDEX))
-                .collect(Collectors.toList());
+            List<Engine.Operation> operations = generateHistoryOnReplica(
+                between(1, 500),
+                randomBoolean(),
+                randomBoolean(),
+                randomBoolean(),
+                Engine.Operation.TYPE.INDEX
+            );
+
             for (Engine.Operation op : operations) {
                 applyOperation(engine, op);
                 applyOperation(nrtEngine, op);

diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java
@@ -1070,14 +1070,29 @@ public List<Engine.Operation> generateHistoryOnReplica(
         boolean allowGapInSeqNo,
         boolean allowDuplicate,
         boolean includeNestedDocs
+    ) throws Exception {
+        return generateHistoryOnReplica(
+            numOps,
+            allowGapInSeqNo,
+            allowDuplicate,
+            includeNestedDocs,
+            randomFrom(Engine.Operation.TYPE.values())
+        );
+    }
+
+    public List<Engine.Operation> generateHistoryOnReplica(
+        int numOps,
+        boolean allowGapInSeqNo,
+        boolean allowDuplicate,
+        boolean includeNestedDocs,
+        Engine.Operation.TYPE opType
     ) throws Exception {
         long seqNo = 0;
         final int maxIdValue = randomInt(numOps * 2);
         final List<Engine.Operation> operations = new ArrayList<>(numOps);
         CheckedBiFunction<String, Integer, ParsedDocument, IOException> nestedParsedDocFactory = nestedParsedDocFactory();
         for (int i = 0; i < numOps; i++) {
             final String id = Integer.toString(randomInt(maxIdValue));
-            final Engine.Operation.TYPE opType = randomFrom(Engine.Operation.TYPE.values());
             final boolean isNestedDoc = includeNestedDocs && opType == Engine.Operation.TYPE.INDEX && randomBoolean();
             final int nestedValues = between(0, 3);
             final long startTime = threadPool.relativeTimeInNanos();