Skip to content

Commit

Permalink
Merge branch 'main' into PassRoleParameterForS3Snapshots
Browse files Browse the repository at this point in the history
  • Loading branch information
@AndreKurait
AndreKurait authored Nov 13, 2024
2 parents 526a75f + c434c64 commit 81a768d
Show file tree
Hide file tree
Showing 31 changed files with 341 additions and 300 deletions.
58 changes: 24 additions & 34 deletions .github/workflows/CI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: CI

on:
push:
pull_request:
pull_request_target:

env:
python-version: '3.11'
Expand Down Expand Up @@ -63,17 +63,14 @@ jobs:
pipenv install --deploy --dev
pipenv run test
pipenv run coverage xml
- name: Upload Coverage with retry
uses: Wandalen/wretry.action@v3.7.2
- name: Upload Coverage
uses: codecov/codecov-action@v4
with:
attempt_limit: 5
attempt_delay: 2000
action: codecov/codecov-action@v4
with: |
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: python-test
fail_ci_if_error: true
fail_ci_if_error: true
files: ./coverage.xml
flags: python-test
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true

console-api-tests:
runs-on: ubuntu-latest
Expand All @@ -90,17 +87,14 @@ jobs:
pipenv install --deploy --dev
pipenv run coverage run --source='.' manage.py test console_api
pipenv run coverage xml
- name: Upload Coverage with retry
uses: Wandalen/wretry.action@v3.7.2
- name: Upload Coverage
uses: codecov/codecov-action@v4
with:
attempt_limit: 5
attempt_delay: 2000
action: codecov/codecov-action@v4
with: |
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage.xml
flags: python-test
fail_ci_if_error: true
fail_ci_if_error: true
files: ./coverage.xml
flags: python-test
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true


gradle-tests:
Expand Down Expand Up @@ -132,19 +126,15 @@ jobs:
path: |
**/build/reports/tests/
**/reports/jacoco/mergedReport/
- name: Upload Coverage with retry
uses: Wandalen/wretry.action@v3.7.2
with:
attempt_limit: 5
attempt_delay: 2000
action: codecov/codecov-action@v4
with: |
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ github.workspace }}/build/reports/jacoco/mergedReport/jacocoMergedReport.xml
flags: gradle-test
fail_ci_if_error: true
disable_search: true
- name: Upload Coverage
uses: codecov/codecov-action@v4
with:
disable_search: true
fail_ci_if_error: true
files: ${{ github.workspace }}/build/reports/jacoco/mergedReport/jacocoMergedReport.xml
flags: gradle-test
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true


python-e2e-tests:
Expand Down
98 changes: 56 additions & 42 deletions MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.opensearch.migrations;

import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -46,29 +47,40 @@ class EndToEndTest {
private File localDirectory;

private static Stream<Arguments> scenarios() {
var scenarios = Stream.<Arguments>builder();

for (var sourceCluster : SupportedClusters.sources()) {
for (var targetCluster : SupportedClusters.targets()) {
for (var command : MetadataCommands.values()) {
scenarios.add(Arguments.of(sourceCluster, targetCluster, TransferMedium.Http, command));
}
// Only test snapshot for migrate case
scenarios.add(Arguments.of(sourceCluster, targetCluster, TransferMedium.SnapshotImage, MetadataCommands.MIGRATE));
}
}

return scenarios.build();
return SupportedClusters.sources().stream()
.flatMap(sourceCluster -> {
// Determine applicable template types based on source version
List<TemplateType> templateTypes = Stream.concat(
Stream.of(TemplateType.Legacy),
(sourceCluster.getVersion().getMajor() >= 7
? Stream.of(TemplateType.Index, TemplateType.IndexAndComponent)
: Stream.empty()))
.collect(Collectors.toList());

return SupportedClusters.targets().stream()
.flatMap(targetCluster -> templateTypes.stream().flatMap(templateType -> {
// Generate arguments for both HTTP and SnapshotImage transfer mediums
Stream<Arguments> httpArgs = Arrays.stream(MetadataCommands.values())
.map(command -> Arguments.of(sourceCluster, targetCluster, TransferMedium.Http, command, templateType));

Stream<Arguments> snapshotArgs = Stream.of(
Arguments.of(sourceCluster, targetCluster, TransferMedium.SnapshotImage, MetadataCommands.MIGRATE, templateType)
);

return Stream.concat(httpArgs, snapshotArgs);
}));
});
}

@ParameterizedTest(name = "From version {0} to version {1}, Command {2}, Medium of transfer {3}")
@ParameterizedTest(name = "From version {0} to version {1}, Command {2}, Medium of transfer {3}, and Template Type {4}")
@MethodSource(value = "scenarios")
void metadataCommand(ContainerVersion sourceVersion, ContainerVersion targetVersion, TransferMedium medium, MetadataCommands command) throws Exception {
void metadataCommand(ContainerVersion sourceVersion, ContainerVersion targetVersion, TransferMedium medium,
MetadataCommands command, TemplateType templateType) {
try (
final var sourceCluster = new SearchClusterContainer(sourceVersion);
final var targetCluster = new SearchClusterContainer(targetVersion)
) {
metadataCommandOnClusters(sourceCluster, targetCluster, medium, command);
metadataCommandOnClusters(sourceCluster, targetCluster, medium, command, templateType);
}
}

Expand All @@ -77,35 +89,35 @@ private enum TransferMedium {
Http
}

private enum TemplateType {
Legacy,
Index,
IndexAndComponent
}

@SneakyThrows
private void metadataCommandOnClusters(
final SearchClusterContainer sourceCluster,
final SearchClusterContainer targetCluster,
final TransferMedium medium,
final MetadataCommands command
final MetadataCommands command,
final TemplateType templateType
) {
// ACTION: Set up the source/target clusters
var bothClustersStarted = CompletableFuture.allOf(
CompletableFuture.runAsync(() -> sourceCluster.start()),
CompletableFuture.runAsync(() -> targetCluster.start())
CompletableFuture.runAsync(sourceCluster::start),
CompletableFuture.runAsync(targetCluster::start)
);
bothClustersStarted.join();

Version sourceVersion = sourceCluster.getContainerVersion().getVersion();
var sourceIsES6_8 = VersionMatchers.isES_6_X.test(sourceVersion);
var sourceIsES7_X = VersionMatchers.isES_7_X.test(sourceVersion) || VersionMatchers.isOS_1_X.test(sourceVersion);

if (!(sourceIsES6_8 || sourceIsES7_X)) {
throw new RuntimeException("This test cannot handle the source cluster version" + sourceVersion);
}

var testData = new TestData();
// Create the component and index templates
var sourceClusterOperations = new ClusterOperations(sourceCluster.getUrl());
if (sourceIsES7_X) {
sourceClusterOperations.createES7Templates(testData.compoTemplateName, testData.indexTemplateName, "author", "blog*");
} else if (sourceIsES6_8) {
sourceClusterOperations.createES6LegacyTemplate(testData.indexTemplateName, "blog*");
if (templateType == TemplateType.Legacy) {
sourceClusterOperations.createLegacyTemplate(testData.indexTemplateName, "blog*");
} else if (templateType == TemplateType.Index) {
sourceClusterOperations.createIndexTemplate(testData.indexTemplateName, "author", "blog*");
} else if (templateType == TemplateType.IndexAndComponent) {
sourceClusterOperations.createComponentTemplate(testData.compoTemplateName, testData.indexTemplateName, "author", "blog*");
}

// Creates a document that uses the template
Expand Down Expand Up @@ -141,7 +153,7 @@ private void metadataCommandOnClusters(
sourceCluster.copySnapshotData(localDirectory.toString());
arguments.fileSystemRepoPath = localDirectory.getAbsolutePath();
arguments.snapshotName = snapshotName;
arguments.sourceVersion = sourceVersion;
arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion();
break;

case Http:
Expand Down Expand Up @@ -171,9 +183,9 @@ private void metadataCommandOnClusters(
result = metadata.evaluate(arguments).execute(metadataContext);
}

verifyCommandResults(result, sourceIsES6_8, testData);
verifyCommandResults(result, templateType, testData);

verifyTargetCluster(targetClusterOperations, command, sourceIsES6_8, testData);
verifyTargetCluster(targetClusterOperations, command, templateType, testData);
}

private static class TestData {
Expand All @@ -188,14 +200,14 @@ private static class TestData {

private void verifyCommandResults(
MigrationItemResult result,
boolean sourceIsES6_8,
TemplateType templateType,
TestData testData) {
log.info(result.asCliOutput());
assertThat(result.getExitCode(), equalTo(0));

var migratedItems = result.getItems();
assertThat(getNames(migratedItems.getIndexTemplates()), containsInAnyOrder(testData.indexTemplateName));
assertThat(getNames(migratedItems.getComponentTemplates()), equalTo(sourceIsES6_8 ? List.of() : List.of(testData.compoTemplateName)));
assertThat(getNames(migratedItems.getComponentTemplates()), equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of()));
assertThat(getNames(migratedItems.getIndexes()), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName, testData.indexThatAlreadyExists));
assertThat(getNames(migratedItems.getAliases()), containsInAnyOrder(testData.aliasInTemplate, testData.aliasName));
}
Expand All @@ -207,7 +219,7 @@ private List<String> getNames(List<CreationResult> items) {
private void verifyTargetCluster(
ClusterOperations targetClusterOperations,
MetadataCommands command,
boolean sourceIsES6_8,
TemplateType templateType,
TestData testData
) {
var expectUpdatesOnTarget = MetadataCommands.MIGRATE.equals(command);
Expand All @@ -233,14 +245,16 @@ private void verifyTargetCluster(
assertThat(res.getValue(), expectUpdatesOnTarget ? verifyAliasWasListed : not(verifyAliasWasListed));

// Check that the templates were migrated
if (sourceIsES6_8) {
if (templateType.equals(TemplateType.Legacy)) {
res = targetClusterOperations.get("/_template/" + testData.indexTemplateName);
assertThat(res.getValue(), res.getKey(), verifyResponseCode);
} else {
} else if(templateType.equals(TemplateType.Index) || templateType.equals(TemplateType.IndexAndComponent)) {
res = targetClusterOperations.get("/_index_template/" + testData.indexTemplateName);
assertThat(res.getValue(), res.getKey(), verifyResponseCode);
var verifyBodyHasComponentTemplate = containsString("composed_of\":[\"" + testData.compoTemplateName + "\"]");
assertThat(res.getValue(), expectUpdatesOnTarget ? verifyBodyHasComponentTemplate : not(verifyBodyHasComponentTemplate));
if (templateType.equals(TemplateType.IndexAndComponent)) {
var verifyBodyHasComponentTemplate = containsString("composed_of\":[\"" + testData.compoTemplateName + "\"]");
assertThat(res.getValue(), expectUpdatesOnTarget ? verifyBodyHasComponentTemplate : not(verifyBodyHasComponentTemplate));
}
}
}
}
Loading

0 comments on commit 81a768d

Please sign in to comment.