[Backport] [1.x] Modernize and consolidate JDKs usage across all stages of the build. Use JDK-17 as bundled JDK distribution to run tests

buildSrc/src/main/java/org/opensearch/gradle/Jdk.java

@@ -49,7 +49,7 @@
 public class Jdk implements Buildable, Iterable<File> {
 
     private static final List<String> ALLOWED_ARCHITECTURES = Collections.unmodifiableList(Arrays.asList("aarch64", "x64"));
-    private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk"));
+    private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk"));
     private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(
         Arrays.asList("darwin", "freebsd", "linux", "mac", "windows")
     );

buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java

@@ -46,7 +46,7 @@
 import org.gradle.api.internal.artifacts.ArtifactAttributes;
 
 public class JdkDownloadPlugin implements Plugin<Project> {
-
+    public static final String VENDOR_ADOPTIUM = "adoptium";
     public static final String VENDOR_ADOPTOPENJDK = "adoptopenjdk";
     public static final String VENDOR_OPENJDK = "openjdk";
 
@@ -108,7 +108,25 @@ private void setupRepository(Project project, Jdk jdk) {
         String repoUrl;
         String artifactPattern;
 
-        if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) {
+        if (jdk.getVendor().equals(VENDOR_ADOPTIUM)) {
+            repoUrl = "https://github.com/adoptium/temurin" + jdk.getMajor() + "-binaries/releases/download/";
+            // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17).
+            // To distinguish between those, the GA releases have only major version component (fe 17+32),
+            // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
+            // version separator '.' should be enough.
+            artifactPattern = "jdk-"
+                + jdk.getBaseVersion()
+                + "+"
+                + jdk.getBuild()
+                + "/OpenJDK"
+                + jdk.getMajor()
+                + (jdk.getBaseVersion().contains(".") ? "U" : "")
+                + "-jdk_[classifier]_[module]_hotspot_"
+                + jdk.getBaseVersion()
+                + "_"
+                + jdk.getBuild()
+                + ".[ext]";
+        } else if (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK)) {
             repoUrl = "https://api.adoptopenjdk.net/v3/binary/version/";
             if (jdk.getMajor().equals("8")) {
                 // legacy pattern for JDK 8
@@ -167,7 +185,7 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
 
     private static String dependencyNotation(Jdk jdk) {
         String platformDep = jdk.getPlatform().equals("darwin") || jdk.getPlatform().equals("mac")
-            ? (jdk.getVendor().equals(VENDOR_ADOPTOPENJDK) ? "mac" : "osx")
+            ? (jdk.getVendor().equals(VENDOR_OPENJDK) ? "osx" : "mac")
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -76,8 +76,8 @@
 public class DistroTestPlugin implements Plugin<Project> {
     private static final String SYSTEM_JDK_VERSION = "8u242+b08";
     private static final String SYSTEM_JDK_VENDOR = "adoptopenjdk";
-    private static final String GRADLE_JDK_VERSION = "14+36@076bab302c7b4508975440c56f6cc26a";
-    private static final String GRADLE_JDK_VENDOR = "openjdk";
+    private static final String GRADLE_JDK_VERSION = "17.0.1+12";
+    private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution
     private static final String EXAMPLE_PLUGIN_CONFIGURATION = "examplePlugin";

buildSrc/src/test/java/org/opensearch/gradle/JdkDownloadPluginTests.java

@@ -60,7 +60,7 @@ public void testUnknownVendor() {
             "11.0.2+33",
             "linux",
             "x64",
-            "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptopenjdk, openjdk]"
+            "unknown vendor [unknown] for jdk [testjdk], must be one of [adoptium, adoptopenjdk, openjdk]"
         );
     }
 

buildSrc/version.properties

@@ -1,8 +1,8 @@
 opensearch        = 1.3.0
 lucene            = 8.10.1
 
-bundled_jdk_vendor = adoptopenjdk
-bundled_jdk = 15.0.1+9
+bundled_jdk_vendor = adoptium
+bundled_jdk = 17.0.1+12
 
 
 
@@ -20,7 +20,7 @@ slf4j             = 1.6.2
 jna               = 5.5.0
 
 netty             = 4.1.72.Final
-joda              = 2.10.4
+joda              = 2.10.12
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)

libs/nio/build.gradle

@@ -27,6 +27,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.opensearch.gradle.info.BuildParams
+
 apply plugin: 'opensearch.publish'
 
 dependencies {
@@ -47,3 +50,10 @@ tasks.named('forbiddenApisMain').configure {
   // es-all is not checked as we connect and accept sockets
   replaceSignatureFiles 'jdk-signatures'
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.nio.channels=ALL-UNNAMED"]
+        jvmArgs += ["--add-opens", "java.base/java.net=ALL-UNNAMED"]
+    }
+}

libs/ssl-config/build.gradle

@@ -27,6 +27,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
+import org.opensearch.gradle.info.BuildParams
+
 apply plugin: "opensearch.publish"
 
 dependencies {
@@ -52,3 +55,10 @@ forbiddenPatterns {
   exclude '**/*.p12'
   exclude '**/*.jks'
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.security.cert=ALL-UNNAMED"]
+    }
+}
+

libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java

@@ -68,10 +68,8 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) {
         assertThat(trustManager.getAcceptedIssuers(), not(emptyArray()));
         // This is a sample of the CAs that we expect on every JRE.
         // We can safely change this list if the JRE's issuer list changes, but we want to assert something useful.
-        assertHasTrustedIssuer(trustManager, "VeriSign");
-        assertHasTrustedIssuer(trustManager, "GeoTrust");
+        // - https://bugs.openjdk.java.net/browse/JDK-8215012: VeriSign, GeoTrust" and "thawte" are gone
         assertHasTrustedIssuer(trustManager, "DigiCert");
-        assertHasTrustedIssuer(trustManager, "thawte");
         assertHasTrustedIssuer(trustManager, "COMODO");
     }
 

plugins/repository-hdfs/build.gradle

@@ -228,6 +228,10 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
           )
         }
       }
+
+      if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED"]
+      }
   }
 
   testClusters."${integTestTaskName}" {

qa/evil-tests/build.gradle

@@ -33,6 +33,8 @@
  * integration, change default filesystem impl, mess with arbitrary
  * threads, etc.
  */
+
+import org.opensearch.gradle.info.BuildParams
 
 apply plugin: 'opensearch.testclusters'
 apply plugin: 'opensearch.standalone-test'
@@ -61,3 +63,9 @@ thirdPartyAudit {
     'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1'
   )
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.lang=ALL-UNNAMED"]
+    }
+}

server/build.gradle

@@ -353,3 +353,9 @@ tasks.named("licenseHeaders").configure {
     excludes << 'org/apache/lucene/search/RegexpQuery87*'
     excludes << 'org/opensearch/client/documentation/placeholder.txt'
 }
+
+tasks.test {
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
+        jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"]
+    }
+}

server/licenses/joda-time-2.10.12.jar.sha1

@@ -0,0 +1 @@
+95b3f193ad0493d94dcd7daa9ea575c30e6be5f5

server/src/test/java/org/opensearch/common/time/DateUtilsTests.java

@@ -58,13 +58,7 @@
 import static org.hamcrest.Matchers.is;
 
 public class DateUtilsTests extends OpenSearchTestCase {
-    private static final Set<String> IGNORE = new HashSet<>(
-        Arrays.asList(
-            "Eire",
-            "Europe/Dublin", // dublin timezone in joda does not account for DST
-            "Asia/Qostanay" // this has been added in joda 2.10.2 but is not part of the JDK 12.0.1 tzdata yet
-        )
-    );
+    private static final Set<String> IGNORE = new HashSet<>(Arrays.asList("Pacific/Enderbury", "Pacific/Kanton", "Pacific/Niue"));
 
     public void testTimezoneIds() {
         assertNull(DateUtils.dateTimeZoneToZoneId(null));

test/fixtures/krb5kdc-fixture/src/main/resources/provision/krb5.conf.template

@@ -43,6 +43,8 @@
     # udp_preference_limit = 1
     kdc_timeout = 3000
     canonicalize = true
+    # See please https://seanjmullan.org/blog/2021/09/14/jdk17 (deprecate 3DES and RC4 in Kerberos)
+    allow_weak_crypto = true
 
 [realms]
     ${REALM_NAME} = {