Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce the number of cluster start/stops for Metadata end to end tests #1186

Merged
merged 1 commit into from
Dec 11, 2024
+20 −36
Merged

Reduce the number of cluster start/stops for Metadata end to end tests #1186

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 20 additions & 36 deletions MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.opensearch.migrations;

import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -48,39 +47,28 @@ class EndToEndTest {

private static Stream<Arguments> scenarios() {
return SupportedClusters.sources().stream()
.flatMap(sourceCluster -> {
// Determine applicable template types based on source version
List<TemplateType> templateTypes = Stream.concat(
Stream.of(TemplateType.Legacy),
(sourceCluster.getVersion().getMajor() >= 7
? Stream.of(TemplateType.Index, TemplateType.IndexAndComponent)
: Stream.empty()))
.collect(Collectors.toList());

return SupportedClusters.targets().stream()
.flatMap(targetCluster -> templateTypes.stream().flatMap(templateType -> {
// Generate arguments for both HTTP and SnapshotImage transfer mediums
Stream<Arguments> httpArgs = Arrays.stream(MetadataCommands.values())
.map(command -> Arguments.of(sourceCluster, targetCluster, TransferMedium.Http, command, templateType));

Stream<Arguments> snapshotArgs = Stream.of(
Arguments.of(sourceCluster, targetCluster, TransferMedium.SnapshotImage, MetadataCommands.MIGRATE, templateType)
);

return Stream.concat(httpArgs, snapshotArgs);
}));
});
.flatMap(sourceCluster ->
SupportedClusters.targets().stream()
.flatMap(targetCluster ->
Stream.concat(
Stream.of(Arguments.of(sourceCluster, targetCluster, TransferMedium.Http)),
Stream.of(Arguments.of(sourceCluster, targetCluster, TransferMedium.SnapshotImage))
)
)
);
}

@ParameterizedTest(name = "From version {0} to version {1}, Command {2}, Medium of transfer {3}, and Template Type {4}")
@ParameterizedTest(name = "From version {0} to version {1}, Command {2}")
@MethodSource(value = "scenarios")
void metadataCommand(ContainerVersion sourceVersion, ContainerVersion targetVersion, TransferMedium medium,
MetadataCommands command, TemplateType templateType) {
void metadataCommand(ContainerVersion sourceVersion, ContainerVersion targetVersion, TransferMedium medium) {
var templateType = VersionMatchers.isES_6_X.test(sourceVersion.getVersion())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where are you testing TemplateType.Legacy on ES 7? There was a bug in that flow that these tests were added thereafter

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you review this matrix for the test cover for template types? I'll update the code to include the correct set of items

ES Version Template Type(s)
ES 6.X Legacy
ES 7.X Legacy & Template & Component Template
OS 1.X Legacy & Template & Component Template
OS 2.X Legacy & Template & Component Template

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that's correct.

There's also a difference from a Template with an inline mapping vs Template linked to a Component Template so they should each be tested.

? TemplateType.Legacy : TemplateType.IndexAndComponent;
try (
final var sourceCluster = new SearchClusterContainer(sourceVersion);
final var targetCluster = new SearchClusterContainer(targetVersion)
) {
metadataCommandOnClusters(sourceCluster, targetCluster, medium, command, templateType);
metadataCommandOnClusters(sourceCluster, targetCluster, medium, MetadataCommands.EVALUATE, templateType);
metadataCommandOnClusters(sourceCluster, targetCluster, medium, MetadataCommands.MIGRATE, templateType);
}
}

Expand All @@ -91,7 +79,6 @@ private enum TransferMedium {

private enum TemplateType {
Legacy,
Index,
IndexAndComponent
}

Expand All @@ -114,9 +101,8 @@ private void metadataCommandOnClusters(
var sourceClusterOperations = new ClusterOperations(sourceCluster.getUrl());
if (templateType == TemplateType.Legacy) {
sourceClusterOperations.createLegacyTemplate(testData.indexTemplateName, "blog*");
} else if (templateType == TemplateType.Index) {
sourceClusterOperations.createIndexTemplate(testData.indexTemplateName, "author", "blog*");
} else if (templateType == TemplateType.IndexAndComponent) {
sourceClusterOperations.createIndexTemplate(testData.indexTemplateName, "author", "blog*");
sourceClusterOperations.createComponentTemplate(testData.compoTemplateName, testData.indexTemplateName, "author", "blog*");
}

Expand All @@ -135,7 +121,7 @@ private void metadataCommandOnClusters(
switch (medium) {
case SnapshotImage:
var snapshotContext = SnapshotTestContext.factory().noOtelTracking();
var snapshotName = "my_snap";
var snapshotName = "my_snap_" + command.name().toLowerCase();
log.info("Source cluster {}", sourceCluster.getUrl());
var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder()
.host(sourceCluster.getUrl())
Expand Down Expand Up @@ -262,13 +248,11 @@ private void verifyTargetCluster(
if (templateType.equals(TemplateType.Legacy)) {
res = targetClusterOperations.get("/_template/" + testData.indexTemplateName);
assertThat(res.getValue(), res.getKey(), verifyResponseCode);
} else if(templateType.equals(TemplateType.Index) || templateType.equals(TemplateType.IndexAndComponent)) {
} else if(templateType.equals(TemplateType.IndexAndComponent)) {
res = targetClusterOperations.get("/_index_template/" + testData.indexTemplateName);
assertThat(res.getValue(), res.getKey(), verifyResponseCode);
if (templateType.equals(TemplateType.IndexAndComponent)) {
var verifyBodyHasComponentTemplate = containsString("composed_of\":[\"" + testData.compoTemplateName + "\"]");
assertThat(res.getValue(), expectUpdatesOnTarget ? verifyBodyHasComponentTemplate : not(verifyBodyHasComponentTemplate));
}
var verifyBodyHasComponentTemplate = containsString("composed_of\":[\"" + testData.compoTemplateName + "\"]");
assertThat(res.getValue(), expectUpdatesOnTarget ? verifyBodyHasComponentTemplate : not(verifyBodyHasComponentTemplate));
}
}
}
Loading