[Backport 2.x] Switch to standard OpenSearch gradle build (opensearch…

…-project#1903)

* Switch to standard OpenSearch gradle build (opensearch-project#1888)

* Rewrote build.gradle to follow OpenSearch plugin standards
  * Enabled license headers check on the repository
  * Did not enable several new repository checks
  * Added maven publishing for security
  * Removed excess forced dependency resolutions
  * Rebuilt projects dependencies, release plugin binary has the exact same dependencies and versions
  * Converted dependencies into runtime dependencies to avoid use during coding
  * Converted dependencies from project wide to test only dependencies
  * jackson-databind version comes from OpenSearch
  * Replaced handmade build manifest with git properties to automated version used by OpenSearch
* Added license headers to files that were missing them
* Checkstyle improvements
  * Disable checkstyle checks that are not errors
  * Moved checkstyle file into directory off of the project root
* Moved standard configuration directory
* Using default test running from OpenSearch
  * Switched to RandomizedTest as the base test class
  * Parameterized test runner cannot be used with RandomizedTest converted test functionality
  * Fixed tests resource issues, added new fields to make this consistent in the codebase
  * Fixed issue with leaky environment variable for security root directory, refactored usage
  * Removed unneeded setDefaultUncaughtExceptionHandler in tests
  * Fixed issues with deprecated internal reflection from Mockito
  * Disabled ThreadLeak detection as it is catching real issues that are mitigated by retries
* Initial pass on non-inclusive terminology, commented out exclusions
* Removed test dependency on scala
* Removed test dependency on legacy xmlsecurity library

Signed-off-by: Peter Nied <petern@amazon.com>
(cherry picked from commit 03a224d)

* Fix build break from cluster manager changes

Signed-off-by: Peter Nied <petern@amazon.com>

Co-authored-by: Peter Nied <petern@amazon.com>

.github/workflows/ci.yml

@@ -150,17 +150,6 @@ jobs:
 
     - run: ./gradlew clean assemble -Dbuild.version_qualifier=${{ env.TEST_QUALIFIER }} && test -s ./build/distributions/opensearch-security-${{ env.SECURITY_PLUGIN_VERSION_ONLY_NUMBER }}-${{ env.TEST_QUALIFIER }}-SNAPSHOT.zip
 
-    - name: Verify updateVersion gradle tasks works
-      env:
-        ExpectedVersionString: "opensearch_version: 2.1.0-SNAPSHOT"
-      run:  |
-        ## Make sure the current doesn't match the test version
-        test "$(./gradlew properties | grep opensearch.version)" != "$ExpectedVersionString"
-        ## Update the new version to 2.1.0
-        ./gradlew clean updateVersion -DnewVersion=2.1.0-SNAPSHOT
-        ## Make sure the version matches expectation
-        test "$(./gradlew properties | grep opensearch.version)" = "$ExpectedVersionString"
-
     - name: List files in the build directory if there was an error
       run: ls -al ./build/distributions/
       if: failure()

build.gradle

@@ -9,23 +9,15 @@
  * GitHub history for details.
  */
 
-
-import com.diffplug.gradle.spotless.JavaExtension
-import org.opensearch.gradle.test.RestIntegTestTask
-
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.1.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
 
         // 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
-
-        common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
-        kafka_version  = '3.0.2'
-
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
         }
@@ -56,8 +48,6 @@ plugins {
     id 'checkstyle'
     id 'nebula.ospackage'  version "8.3.0"
     id "org.gradle.test-retry" version "1.3.1"
-    id 'eclipse'
-    id "com.github.spotbugs" version "5.0.13"
 }
 
 allprojects {
@@ -67,8 +57,6 @@ allprojects {
 
 apply plugin: 'opensearch.opensearchplugin'
 apply plugin: 'opensearch.pluginzip'
-apply plugin: 'opensearch.rest-test'
-apply plugin: 'opensearch.testclusters'
 
 licenseFile = rootProject.file('LICENSE.txt')
 noticeFile = rootProject.file('NOTICE.txt')
@@ -77,23 +65,9 @@ spotless {
   java {
     // note: you can use an empty string for all the imports you didn't specify explicitly, and '\\#` prefix for static imports
     importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#')
-    targetExclude('src/integrationTest/**')
-  }
-  format("integrationTest", JavaExtension) {
-      target('src/integrationTest/java/**/*.java')
-      importOrder('java', 'javax', '', 'com.amazon', 'org.opensearch', '\\#')
-      indentWithTabs(4)
   }
 }
 
-spotbugs {
-    includeFilter = file('spotbugs-include.xml')
-}
-
-spotbugsTest {
-    enabled = false
-}
-
 java.sourceCompatibility = JavaVersion.VERSION_11
 java.targetCompatibility = JavaVersion.VERSION_11
 
@@ -108,20 +82,11 @@ forbiddenApisTest.enabled = false
 filepermissions.enabled = false
 forbiddenPatterns.enabled = false
 testingConventions.enabled = false
-// Conflicts between runtime kafka-clients:x.y.z & testRuntime kafka-clients:x.y.z:test
+// Conflicts between runtime kafka-clients:3.0.1 & testRuntime kafka-clients:3.0.1:test
 jarHell.enabled = false
-tasks.whenTaskAdded {task ->
-    if(task.name.contains("forbiddenApisIntegrationTest")) {
-        task.enabled = false
-    }
-}
-
 
 test {
     include '**/*.class'
-    filter {
-        excludeTestsMatching "org.opensearch.security.sanity.tests.*"
-    }
     maxParallelForks = 8
     jvmArgs += "-Xmx3072m"
     if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
@@ -188,7 +153,6 @@ publishing {
             pom {
                 name = "opensearch-security"
                 description = "Provide access control related features for OpenSearch"
-                groupId = "org.opensearch.plugin"
                 licenses {
                     license {
                         name = "The Apache License, Version 2.0"
@@ -246,82 +210,40 @@ bundlePlugin {
     }
 }
 
-configurations {
-    all {
-        resolutionStrategy {
-            force 'commons-codec:commons-codec:1.14'
-            force 'org.slf4j:slf4j-api:1.7.30'
-            force 'org.scala-lang:scala-library:2.13.9'
-            force 'commons-io:commons-io:2.11.0'
-            force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
-            force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-            force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
-            force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-            force "io.netty:netty-buffer:${versions.netty}"
-            force "io.netty:netty-common:${versions.netty}"
-            force "io.netty:netty-handler:${versions.netty}"
-            force "io.netty:netty-transport:${versions.netty}"
-            force "io.netty:netty-transport-native-unix-common:${versions.netty}"
-        }
-    }
-
-    integrationTestImplementation.extendsFrom implementation
-    integrationTestRuntimeOnly.extendsFrom runtimeOnly
-}
-
-//create source set 'integrationTest'
-//add classes from the main source set to the compilation and runtime classpaths of the integrationTest
-sourceSets {
-    integrationTest {
-        java {
-            srcDir file ('src/integrationTest/java')
-            compileClasspath += sourceSets.main.output
-            runtimeClasspath += sourceSets.main.output
-        }
-        resources {
-            srcDir file('src/integrationTest/resources')
-        }
-        processIntegrationTestResources {
-            duplicatesStrategy(DuplicatesStrategy.INCLUDE)
-        }
+configurations.all {
+    resolutionStrategy {
+        force 'commons-codec:commons-codec:1.14'
+        force 'org.slf4j:slf4j-api:1.7.30'
+        force 'org.scala-lang:scala-library:2.13.8'
+        force 'commons-io:commons-io:2.11.0'
+        force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
+        force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+        force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
+        force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+        force 'io.netty:netty-buffer:4.1.73.Final'
+        force 'io.netty:netty-common:4.1.73.Final'
+        force 'io.netty:netty-handler:4.1.73.Final'
+        force 'io.netty:netty-transport:4.1.73.Final'
     }
 }
 
-//add new task that runs integration tests
-task integrationTest(type: Test) {
-    description = 'Run integration tests.'
-    group = 'verification'
-    systemProperty "java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"
-    testClassesDirs = sourceSets.integrationTest.output.classesDirs
-    classpath = sourceSets.integrationTest.runtimeClasspath
-
-    //run the integrationTest task after the test task
-    shouldRunAfter test
-}
-
-//run the integrationTest task before the check task
-check.dependsOn integrationTest
-
 dependencies {
     implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5'
     implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
     implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
-    implementation "org.apache.httpcomponents.client5:httpclient5-cache:${versions.httpclient5}"
-    implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
-    implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-    implementation "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
     implementation 'com.google.guava:guava:30.0-jre'
     implementation 'org.greenrobot:eventbus:3.2.0'
     implementation 'commons-cli:commons-cli:1.3.1'
-    implementation "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
+    implementation 'org.bouncycastle:bcprov-jdk15on:1.67'
     implementation 'org.ldaptive:ldaptive:1.2.3'
+    implementation 'org.apache.httpcomponents:httpclient-cache:4.5.13'
     implementation 'io.jsonwebtoken:jjwt-api:0.10.8'
     implementation('org.apache.cxf:cxf-rt-rs-security-jose:3.4.5') {
         exclude(group: 'jakarta.activation', module: 'jakarta.activation-api')
     }
     implementation 'com.github.wnameless:json-flattener:0.5.0'
     implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.4'
-    implementation "org.apache.kafka:kafka-clients:${kafka_version}"
+    implementation 'org.apache.kafka:kafka-clients:3.0.1'
     implementation 'com.onelogin:java-saml:2.5.0'
     implementation 'com.onelogin:java-saml-core:2.5.0'
 
@@ -361,11 +283,13 @@ dependencies {
     implementation 'commons-lang:commons-lang:2.4'
     implementation 'commons-collections:commons-collections:3.2.2'
     implementation 'com.jayway.jsonpath:json-path:2.4.0'
+    implementation 'org.apache.httpcomponents:httpclient:4.5.13'
+    implementation 'org.apache.httpcomponents:httpclient:4.5.13'
     implementation 'net.minidev:json-smart:2.4.7'
     runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.10.8'
     runtimeOnly 'io.jsonwebtoken:jjwt-jackson:0.10.8'
     runtimeOnly 'com.google.guava:failureaccess:1.0.1'
-    runtimeOnly 'org.apache.commons:commons-text:1.10.0'
+    runtimeOnly 'org.apache.commons:commons-text:1.2'
     runtimeOnly 'org.glassfish.jaxb:jaxb-runtime:2.3.4'
     runtimeOnly 'com.google.j2objc:j2objc-annotations:1.3'
     runtimeOnly 'com.google.code.findbugs:jsr305:3.0.2'
@@ -376,34 +300,31 @@ dependencies {
     runtimeOnly 'org.xerial.snappy:snappy-java:1.1.8.1'
     runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.1'
     runtimeOnly 'org.glassfish.jaxb:txw2:2.3.4'
-    runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.4.0'
+    runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.2.6'
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.2.5'
     runtimeOnly 'org.apache.santuario:xmlsec:2.2.3'
     runtimeOnly 'com.github.luben:zstd-jni:1.5.0-2'
     runtimeOnly 'org.checkerframework:checker-qual:3.5.0'
-    runtimeOnly "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
 
 
     implementation 'org.apache.commons:commons-lang3:3.4'
-    testImplementation "org.opensearch:common-utils:${common_utils_version}"
     testImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
     testImplementation "org.opensearch:opensearch-ssl-config:${opensearch_version}"
     testImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
     testImplementation "org.opensearch.plugin:lang-mustache-client:${opensearch_version}"
     testImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}"
     testImplementation "org.opensearch.plugin:aggs-matrix-stats-client:${opensearch_version}"
     testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1'
-    testImplementation 'commons-io:commons-io:2.11.0'
+    testImplementation 'commons-io:commons-io:2.7'
     testImplementation 'javax.servlet:servlet-api:2.5'
     testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.9'
     testImplementation 'com.github.stephenc.jcip:jcip-annotations:1.0-1'
     testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.9'
     testImplementation 'javax.servlet:servlet-api:2.5'
     testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.13'
-    testImplementation "org.apache.httpcomponents.client5:httpclient5-fluent:${versions.httpclient5}"
-    testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}"
-    testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}:test"
-    testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test"
+    testImplementation 'org.apache.kafka:kafka_2.13:3.0.1'
+    testImplementation 'org.apache.kafka:kafka_2.13:3.0.1:test'
+    testImplementation 'org.apache.kafka:kafka-clients:3.0.1:test'
     testImplementation 'org.springframework.kafka:spring-kafka-test:2.8.6'
     testImplementation 'org.springframework:spring-beans:5.3.20'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2'
@@ -415,37 +336,19 @@ dependencies {
     testRuntimeOnly ('org.springframework:spring-core:5.3.21') {
         exclude(group:'org.springframework', module: 'spring-jcl' )
     }
-    testRuntimeOnly 'org.scala-lang:scala-library:2.13.9'
+    testRuntimeOnly 'org.scala-lang:scala-library:2.13.8'
     testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0'
     testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
-    testRuntimeOnly 'org.apache.zookeeper:zookeeper:3.7.1'
-    testRuntimeOnly "org.apache.kafka:kafka-metadata:${kafka_version}"
-    testRuntimeOnly "org.apache.kafka:kafka-storage:${kafka_version}"
+    testRuntimeOnly 'org.apache.zookeeper:zookeeper:3.6.3'
+    testRuntimeOnly 'org.apache.kafka:kafka-metadata:3.0.1'
+    testRuntimeOnly 'org.apache.kafka:kafka-storage:3.0.1'
 
 
 
     implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
     implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
 
     compileOnly "org.opensearch:opensearch:${opensearch_version}"
-
-    //integration test framework:
-    integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.7.1') {
-        exclude(group: 'junit', module: 'junit')
-    }
-    integrationTestImplementation 'junit:junit:4.13.2'
-    integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
-    integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
-    integrationTestImplementation 'commons-io:commons-io:2.11.0'
-    integrationTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1'
-    integrationTestImplementation 'org.apache.logging.log4j:log4j-jul:2.17.1'
-    integrationTestImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.1'
-    integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
-    integrationTestImplementation "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
-    integrationTestImplementation "org.bouncycastle:bcutil-jdk15on:${versions.bouncycastle}"
-    integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
-        exclude(group: 'org.hamcrest', module: 'hamcrest')
-    }
 }
 
 jar {
@@ -521,23 +424,6 @@ publishing {
     }
 }
 
-task integTestRemote(type: RestIntegTestTask) {
-
-    systemProperty "tests.security.manager", "false"
-    systemProperty "user", System.getProperty("user")
-    systemProperty "password", System.getProperty("password")
-    systemProperty "https", System.getProperty("https")
-    systemProperty "security.enabled", "true"
-
-    filter {
-        setIncludePatterns("org.opensearch.security.sanity.tests.*IT")
-    }
-}
-
-integTestRemote.enabled = System.getProperty("tests.rest.cluster") != null
-// should be updated appropriately, when we add integTests in future
-integTest.enabled = false
-
 // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
 afterEvaluate {
     ospackage {
@@ -600,20 +486,3 @@ afterEvaluate {
         tasks = ['build', 'buildRpm', 'buildDeb']
     }
 }
-
-// updateVersion: Task to auto increment to the next development iteration
-task updateVersion {
-    onlyIf { System.getProperty('newVersion') }
-    doLast {
-        ext.newVersion = System.getProperty('newVersion')
-        println "Setting version to ${newVersion}."
-        // String tokenization to support -SNAPSHOT
-        ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) {
-            fileset(dir: projectDir) {
-                // Include the required files that needs to be updated with new Version
-                include(name: "bwc-test/build.gradle")
-            }
-        }
-        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
-    }
-}

checkstyle/sun_checks.xml

@@ -201,7 +201,7 @@
     <property name="format" value="master"/>
     <property name="ignoreCase" value="true"/>
     <property name="message" value="Usage should be switched to cluster manager" />
-    <property name="severity" value="error"/>
+    <property name="severity" value="ignore"/>
   </module>
 
   <module name="SuppressWithPlainTextCommentFilter">