Skip to content

Commit

Permalink
Merge pull request opensearch-project#1195 from AndreKurait/EnableMul…
Browse files Browse the repository at this point in the history 
…tiTypeGHA

Enable MultiTypeMappingTransformationTest
  • Loading branch information
@AndreKurait
AndreKurait authored Dec 11, 2024
2 parents 3eb8c08 + e41394c commit ac5dd29
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 78 deletions.
38 changes: 0 additions & 38 deletions .github/workflows/CI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,44 +111,6 @@ jobs:
with:
gradle-version: ${{ env.gradle-version }}
gradle-home-cache-cleanup: true
- name: Generate Cache Key from Dockerfiles
id: generate_cache_key
run: |
files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \))
file_contents=$(cat $files)
key=$(echo "${file_contents}" | sha1sum | awk '{print $1}')
echo "key=${key}" >> "$GITHUB_OUTPUT"
- name: Cache Docker Images
uses: ScribeMD/docker-cache@0.5.0
with:
key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
- name: Pre pull images
run: |
pull_if_not_present() {
local image="$1"
if docker image inspect "$image" > /dev/null 2>&1; then
echo "Image '$image' already exists locally."
else
echo "Pulling image '$image'..."
docker pull "$image"
fi
}
images=(
"opensearchproject/opensearch:1.3.16"
"opensearchproject/opensearch:2.14.0"
"docker.elastic.co/elasticsearch/elasticsearch:7.17.22"
"docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2"
"docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23"
"docker.elastic.co/elasticsearch/elasticsearch:5.6.16"
"httpd:alpine"
"confluentinc/cp-kafka:7.5.0"
"ghcr.io/shopify/toxiproxy:latest"
"amazonlinux:2023"
"alpine:3.16"
)
for image in "${images[@]}"; do
pull_if_not_present "$image"
done
- name: Run Gradle Build
run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
env:
Expand Down
6 changes: 3 additions & 3 deletions MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.opensearch.migrations;

import java.io.File;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.CompletableFuture;

Expand All @@ -26,7 +26,7 @@
abstract class BaseMigrationTest {

@TempDir
protected File localDirectory;
protected Path localDirectory;

@Getter
protected SearchClusterContainer sourceCluster;
Expand Down Expand Up @@ -83,7 +83,7 @@ protected String createSnapshot(String snapshotName) {
*/
protected MigrateOrEvaluateArgs prepareSnapshotMigrationArgs(String snapshotName) {
var arguments = new MigrateOrEvaluateArgs();
arguments.fileSystemRepoPath = localDirectory.getAbsolutePath();
arguments.fileSystemRepoPath = localDirectory.toString();
arguments.snapshotName = snapshotName;
arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion();
arguments.targetArgs.host = targetCluster.getUrl();
Expand Down
75 changes: 38 additions & 37 deletions ...Migration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.BindMode;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
Expand All @@ -31,38 +29,41 @@ class MultiTypeMappingTransformationTest extends BaseMigrationTest {

@SneakyThrows
@Test
@Disabled("TODO, fix in GHA")
public void multiTypeTransformationTest_union() {
var es5Repo = "es5";
var snapshotName = "es5-created-index";
var originalIndexName = "test_index";

try (
final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16);
final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23)
.withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, BindMode.READ_WRITE);
final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0)
final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16)
) {
indexCreatedCluster.start();

this.sourceCluster = upgradedSourceCluster;
this.targetCluster = targetCluster;

startClusters();

var indexCreatedOperations = new ClusterOperations(indexCreatedCluster.getUrl());
var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl());

var originalIndexName = "test_index";

// Create index and add documents on the source cluster
indexCreatedOperations.createIndex(originalIndexName);
indexCreatedOperations.createDocument(originalIndexName, "1", "{\"field1\":\"My Name\"}", null, "type1");
indexCreatedOperations.createDocument(originalIndexName, "2", "{\"field1\":\"string\", \"field2\":123}", null, "type2");
indexCreatedOperations.createDocument(originalIndexName, "3", "{\"field3\":1.1}", null, "type3");

var snapshotName = "es5-created-index";
var es5Repo = "es5";
indexCreatedOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo);
indexCreatedOperations.takeSnapshot(es5Repo, snapshotName, originalIndexName);
indexCreatedCluster.copySnapshotData(localDirectory.toString());
}

indexCreatedCluster.copySnapshotData(localDirectory.getAbsolutePath());
try (
final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23);
final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0)
) {
this.sourceCluster = upgradedSourceCluster;
this.targetCluster = targetCluster;

startClusters();

upgradedSourceCluster.putSnapshotData(localDirectory.toString());

var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl());

// Register snapshot repository and restore snapshot in ES 6 cluster
upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo);
Expand Down Expand Up @@ -131,25 +132,25 @@ public void es5_doesNotAllow_multiTypeConflicts() {

var originalIndexName = "test_index";
String body = "{" +
" \"settings\": {" +
" \"index\": {" +
" \"number_of_shards\": 5," +
" \"number_of_replicas\": 0" +
" }" +
" }," +
" \"mappings\": {" +
" \"type1\": {" +
" \"properties\": {" +
" \"field1\": { \"type\": \"float\" }" +
" }" +
" }," +
" \"type2\": {" +
" \"properties\": {" +
" \"field1\": { \"type\": \"long\" }" +
" }" +
" }" +
" }" +
"}";
" \"settings\": {" +
" \"index\": {" +
" \"number_of_shards\": 5," +
" \"number_of_replicas\": 0" +
" }" +
" }," +
" \"mappings\": {" +
" \"type1\": {" +
" \"properties\": {" +
" \"field1\": { \"type\": \"float\" }" +
" }" +
" }," +
" \"type2\": {" +
" \"properties\": {" +
" \"field1\": { \"type\": \"long\" }" +
" }" +
" }" +
" }" +
"}";
var res = clusterOperations.put("/" + originalIndexName, body);
assertThat(res.getKey(), equalTo(400));
assertThat(res.getValue(), containsString("mapper [field1] cannot be changed from type [long] to [float]"));
Expand Down
11 changes: 11 additions & 0 deletions ...estFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.MountableFile;

/**
* Containerized version of Elasticsearch cluster
Expand Down Expand Up @@ -112,6 +113,16 @@ public void copySnapshotData(final String directory) {
}
}

public void putSnapshotData(final String directory) {
try {
this.copyFileToContainer(MountableFile.forHostPath(directory), CLUSTER_SNAPSHOT_DIR);
this.execInContainer("chown", "-R", "elasticsearch:elasticsearch", CLUSTER_SNAPSHOT_DIR);
} catch (final Exception e) {
throw new RuntimeException(e);
}
}


public void start() {
log.info("Starting container version:" + containerVersion.version);
super.start();
Expand Down

0 comments on commit ac5dd29

Please sign in to comment.