From 5ef449a7fd24fc575c481aede09d21f7d4ffd457 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 11 Dec 2024 16:12:06 -0600 Subject: [PATCH 1/2] Remove docker caching from gradle tests Signed-off-by: Andre Kurait --- .github/workflows/CI.yml | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d3c24a105..8a2ae206d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -111,44 +111,6 @@ jobs: with: gradle-version: ${{ env.gradle-version }} gradle-home-cache-cleanup: true - - name: Generate Cache Key from Dockerfiles - id: generate_cache_key - run: | - files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \)) - file_contents=$(cat $files) - key=$(echo "${file_contents}" | sha1sum | awk '{print $1}') - echo "key=${key}" >> "$GITHUB_OUTPUT" - - name: Cache Docker Images - uses: ScribeMD/docker-cache@0.5.0 - with: - key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} - - name: Pre pull images - run: | - pull_if_not_present() { - local image="$1" - if docker image inspect "$image" > /dev/null 2>&1; then - echo "Image '$image' already exists locally." - else - echo "Pulling image '$image'..." - docker pull "$image" - fi - } - images=( - "opensearchproject/opensearch:1.3.16" - "opensearchproject/opensearch:2.14.0" - "docker.elastic.co/elasticsearch/elasticsearch:7.17.22" - "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2" - "docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23" - "docker.elastic.co/elasticsearch/elasticsearch:5.6.16" - "httpd:alpine" - "confluentinc/cp-kafka:7.5.0" - "ghcr.io/shopify/toxiproxy:latest" - "amazonlinux:2023" - "alpine:3.16" - ) - for image in "${images[@]}"; do - pull_if_not_present "$image" - done - name: Run Gradle Build run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace env: From e41394cce9adb657890f2e7c179c666994613fa0 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 11 Dec 2024 16:50:03 -0600 Subject: [PATCH 2/2] Simplify snapshot and restore in MultiTypeMappingTransformationTest Signed-off-by: Andre Kurait --- .../migrations/BaseMigrationTest.java | 6 +- .../MultiTypeMappingTransformationTest.java | 75 ++++++++++--------- .../framework/SearchClusterContainer.java | 11 +++ 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java index b8236de7a..03451edf8 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java @@ -1,6 +1,6 @@ package org.opensearch.migrations; -import java.io.File; +import java.nio.file.Path; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -26,7 +26,7 @@ abstract class BaseMigrationTest { @TempDir - protected File localDirectory; + protected Path localDirectory; @Getter protected SearchClusterContainer sourceCluster; @@ -83,7 +83,7 @@ protected String createSnapshot(String snapshotName) { */ protected MigrateOrEvaluateArgs prepareSnapshotMigrationArgs(String snapshotName) { var arguments = new MigrateOrEvaluateArgs(); - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); + arguments.fileSystemRepoPath = localDirectory.toString(); arguments.snapshotName = snapshotName; arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); arguments.targetArgs.host = targetCluster.getUrl(); diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java index e97ffda4d..6473be7fc 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java @@ -12,10 +12,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.BindMode; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; @@ -31,25 +29,17 @@ class MultiTypeMappingTransformationTest extends BaseMigrationTest { @SneakyThrows @Test - @Disabled("TODO, fix in GHA") public void multiTypeTransformationTest_union() { + var es5Repo = "es5"; + var snapshotName = "es5-created-index"; + var originalIndexName = "test_index"; + try ( - final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16); - final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23) - .withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, BindMode.READ_WRITE); - final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0) + final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16) ) { indexCreatedCluster.start(); - this.sourceCluster = upgradedSourceCluster; - this.targetCluster = targetCluster; - - startClusters(); - var indexCreatedOperations = new ClusterOperations(indexCreatedCluster.getUrl()); - var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl()); - - var originalIndexName = "test_index"; // Create index and add documents on the source cluster indexCreatedOperations.createIndex(originalIndexName); @@ -57,12 +47,23 @@ public void multiTypeTransformationTest_union() { indexCreatedOperations.createDocument(originalIndexName, "2", "{\"field1\":\"string\", \"field2\":123}", null, "type2"); indexCreatedOperations.createDocument(originalIndexName, "3", "{\"field3\":1.1}", null, "type3"); - var snapshotName = "es5-created-index"; - var es5Repo = "es5"; indexCreatedOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo); indexCreatedOperations.takeSnapshot(es5Repo, snapshotName, originalIndexName); + indexCreatedCluster.copySnapshotData(localDirectory.toString()); + } - indexCreatedCluster.copySnapshotData(localDirectory.getAbsolutePath()); + try ( + final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23); + final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0) + ) { + this.sourceCluster = upgradedSourceCluster; + this.targetCluster = targetCluster; + + startClusters(); + + upgradedSourceCluster.putSnapshotData(localDirectory.toString()); + + var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl()); // Register snapshot repository and restore snapshot in ES 6 cluster upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo); @@ -131,25 +132,25 @@ public void es5_doesNotAllow_multiTypeConflicts() { var originalIndexName = "test_index"; String body = "{" + - " \"settings\": {" + - " \"index\": {" + - " \"number_of_shards\": 5," + - " \"number_of_replicas\": 0" + - " }" + - " }," + - " \"mappings\": {" + - " \"type1\": {" + - " \"properties\": {" + - " \"field1\": { \"type\": \"float\" }" + - " }" + - " }," + - " \"type2\": {" + - " \"properties\": {" + - " \"field1\": { \"type\": \"long\" }" + - " }" + - " }" + - " }" + - "}"; + " \"settings\": {" + + " \"index\": {" + + " \"number_of_shards\": 5," + + " \"number_of_replicas\": 0" + + " }" + + " }," + + " \"mappings\": {" + + " \"type1\": {" + + " \"properties\": {" + + " \"field1\": { \"type\": \"float\" }" + + " }" + + " }," + + " \"type2\": {" + + " \"properties\": {" + + " \"field1\": { \"type\": \"long\" }" + + " }" + + " }" + + " }" + + "}"; var res = clusterOperations.put("/" + originalIndexName, body); assertThat(res.getKey(), equalTo(400)); assertThat(res.getValue(), containsString("mapper [field1] cannot be changed from type [long] to [float]")); diff --git a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java index ec185ba96..787232ad9 100644 --- a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java +++ b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java @@ -14,6 +14,7 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; /** * Containerized version of Elasticsearch cluster @@ -112,6 +113,16 @@ public void copySnapshotData(final String directory) { } } + public void putSnapshotData(final String directory) { + try { + this.copyFileToContainer(MountableFile.forHostPath(directory), CLUSTER_SNAPSHOT_DIR); + this.execInContainer("chown", "-R", "elasticsearch:elasticsearch", CLUSTER_SNAPSHOT_DIR); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + public void start() { log.info("Starting container version:" + containerVersion.version); super.start();