Add windows & mac build (#591)

Signed-off-by: Ankit Kala <ankikala@amazon.com> Signed-off-by: Ankit Kala <ankikala@amazon.com> (cherry picked from commit d12271a)
opensearch-project · Dec 5, 2022 · d372b17 · d372b17
1 parent 98cf601
commit d372b17
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 33 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -0,0 +1,52 @@
+name: CCR Test Workflow
+# This workflow is triggered on pull requests to main branch
+on:
+  pull_request:
+    branches:
+      - '*'
+  push:
+    branches:
+      - '*'
+
+# We build for all combinations but run tests only on one combination (linux & latest java)
+jobs:
+  build:
+    continue-on-error: true
+    strategy:
+      matrix:
+        java:
+          - 8
+          - 11
+          - 14
+    # Job name
+    name: Run integration tests on linux with Java ${{ matrix.java }}
+    runs-on: ubuntu-latest
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Build and run Replication tests
+        run: |
+          ./gradlew clean release -D"build.snapshot=true"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: |
+            build/testclusters/integTest-*/logs/*
+            build/testclusters/leaderCluster-*/logs/*
+            build/testclusters/followCluster-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p cross-cluster-replication-artifacts
+          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
+      - name: Uploads coverage
+        with:
+          fetch-depth: 2
+        uses: codecov/codecov-action@v1.2.1
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,4 +1,4 @@
-name: Test and Build Workflow
+name: Build Replication plugin
 # This workflow is triggered on pull requests to main branch
 on:
   pull_request:
@@ -8,17 +8,18 @@ on:
     branches:
       - '*'
 
+# We build for other platforms except linux which is already covered in build-and-test.
+# Also, We're not running tests here as those are already covered with linux build.
 jobs:
   build:
+    continue-on-error: true
     strategy:
       matrix:
         java: 
-          - 8
-          - 11
           - 14
     # Job name
-    name: Build Replication plugin
-    runs-on: ubuntu-latest
+    name: Java ${{ matrix.java }} On ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -30,21 +31,4 @@ jobs:
         uses: actions/checkout@v2
       - name: Build and run Replication tests
         run: |
-          ./gradlew clean release -Dbuild.snapshot=true
-      - name: Upload failed logs
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: logs
-          path: |
-            build/testclusters/integTest-*/logs/*
-            build/testclusters/leaderCluster-*/logs/*
-            build/testclusters/followCluster-*/logs/*
-      - name: Create Artifact Path
-        run: |
-          mkdir -p cross-cluster-replication-artifacts
-          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
-      - name: Uploads coverage
-        with:
-          fetch-depth: 2
-        uses: codecov/codecov-action@v1.2.1
+          ./gradlew clean release -D"build.snapshot=true" -x test -x IntegTest
diff --git a/.github/workflows/security-tests.yml b/.github/workflows/security-tests.yml
@@ -33,7 +33,7 @@ jobs:
         working-directory: ./security
         run: |
           ./gradlew clean build -Dbuild.snapshot=false -x test
-          cp build/distributions/opensearch-security-1.3.2.0.zip ../src/test/resources/security/plugin/opensearch-security.zip
+          cp build/distributions/opensearch-security-1.3.7.0.zip ../src/test/resources/security/plugin/opensearch-security.zip
       - name: Build and run Replication tests
         run: |
           ls -al src/test/resources/security/plugin

diff --git a/build.gradle b/build.gradle
@@ -33,7 +33,7 @@ import org.opensearch.gradle.test.RestIntegTestTask
 buildscript {
     ext {
         isSnapshot = "true" == System.getProperty("build.snapshot", "false")
-        opensearch_version = System.getProperty("opensearch.version", "1.3.2-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "1.3.7-SNAPSHOT")
         // Taken from https://github.com/opensearch-project/alerting/blob/main/build.gradle#L33
         // 1.0.0 -> 1.0.0.0, and 1.0.0-SNAPSHOT -> 1.0.0.0-SNAPSHOT
         opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')
@@ -43,7 +43,7 @@ buildscript {
         plugin_previous_version = opensearch_previous_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')
 
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-        kotlin_version = System.getProperty("kotlin.version", "1.3.72")
+        kotlin_version = System.getProperty("kotlin.version", "1.6.0")
 
     }
 
@@ -59,7 +59,7 @@ buildscript {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
         classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
         classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.0.0-RC15"
-        classpath "org.jacoco:org.jacoco.agent:0.8.5"
+        classpath "org.jacoco:org.jacoco.agent:0.8.7"
     }
 }
 
@@ -75,6 +75,9 @@ allprojects {
     if (isSnapshot) {
         version += "-SNAPSHOT"
     }
+    // Have resolve the jacoco version here to work with kotlin
+    // Ref: https://github.com/jacoco/jacoco/issues/1187
+    jacoco.toolVersion = "0.8.7"
 }
 
 apply plugin: 'java'
@@ -102,7 +105,7 @@ configurations.all {
         force 'org.apache.httpcomponents:httpclient-osgi:4.5.13'
         force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
         force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
-        force 'com.fasterxml.jackson.core:jackson-databind:2.13.4'
+        force 'com.fasterxml.jackson.core:jackson-databind:2.13.4.2'
         force 'org.yaml:snakeyaml:1.32'
         force 'org.codehaus.plexus:plexus-utils:3.0.24'
     }

diff --git a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt
@@ -91,7 +91,9 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp
                         !leaderSettings.get(ReplicationPlugin.REPLICATED_INDEX_SETTING.key).isNullOrBlank()) {
                     throw IllegalArgumentException("Cannot Replicate a Replicated Index ${request.leaderIndex}")
                 }
-                if (!leaderSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.key, true)) {
+
+                // Soft deletes should be enabled for replication to work.
+                if (!leaderSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.key, false)) {
                     throw IllegalArgumentException("Cannot Replicate an index where the setting ${IndexSettings.INDEX_SOFT_DELETES_SETTING.key} is disabled")
                 }
 
@@ -127,9 +129,19 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp
 
     private suspend fun getLeaderIndexSettings(leaderAlias: String, leaderIndex: String): Settings {
         val remoteClient = client.getRemoteClusterClient(leaderAlias)
-        val getSettingsRequest = GetSettingsRequest().includeDefaults(false).indices(leaderIndex)
-        val settingsResponse = remoteClient.suspending(remoteClient.admin().indices()::getSettings,
-                injectSecurityContext = true)(getSettingsRequest)
-        return settingsResponse.indexToSettings.get(leaderIndex) ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}")
+        val getSettingsRequest = GetSettingsRequest().includeDefaults(true).indices(leaderIndex)
+        val settingsResponse = remoteClient.suspending(
+            remoteClient.admin().indices()::getSettings,
+            injectSecurityContext = true
+        )(getSettingsRequest)
+
+        val leaderSettings = settingsResponse.indexToSettings.get(leaderIndex)
+            ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}")
+        val leaderDefaultSettings = settingsResponse.indexToDefaultSettings.get(leaderIndex)
+            ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}")
+
+        // Since we want user configured as well as default settings, we combine both by putting default settings
+        // and then the explicitly set ones to override the default settings.
+        return Settings.builder().put(leaderDefaultSettings).put(leaderSettings).build()
     }
 }
diff --git a/src/main/kotlin/org/opensearch/replication/task/CrossClusterReplicationTask.kt b/src/main/kotlin/org/opensearch/replication/task/CrossClusterReplicationTask.kt
@@ -21,6 +21,7 @@ import org.opensearch.replication.util.suspending
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.NonCancellable
+import kotlinx.coroutines.ObsoleteCoroutinesApi
 import kotlinx.coroutines.cancel
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.isActive
@@ -178,6 +179,7 @@ abstract class CrossClusterReplicationTask(id: Long, type: String, action: Strin
         client.suspending(::updatePersistentTaskState)(state)
     }
 
+    @ObsoleteCoroutinesApi
     protected abstract suspend fun execute(scope: CoroutineScope, initialState: PersistentTaskState?)
 
     protected open suspend fun cleanup() {}