Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix e2e acks test #3471

Merged
merged 8 commits into from
Oct 11, 2023
Merged

Fix e2e acks test #3471

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3045967
Disable flaky e2e acks test
kkondaka Oct 10, 2023
7f1fb70
Disabled another flaky test
kkondaka Oct 10, 2023
cba386f
Added debug statements to debug the failing tests
kkondaka Oct 10, 2023
77eda60
Modified to assign unique names to pipelines
kkondaka Oct 10, 2023
93f0913
Trying with enabling the disabled test
kkondaka Oct 10, 2023
d86f3b3
Fixed failing checkstyle error
kkondaka Oct 10, 2023
ce2f66c
Reduced sleep time in InMemorySource
kkondaka Oct 10, 2023
45c9483
Modified to use log4j
kkondaka Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions .../src/integrationTest/java/org/opensearch/dataprepper/integration/PipelinesWithAcksIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.util.List;
import java.util.concurrent.TimeUnit;
import java.time.Instant;

import static org.awaitility.Awaitility.await;
import static org.hamcrest.CoreMatchers.equalTo;
Expand Down Expand Up @@ -44,13 +45,15 @@ void setUp(String configFile) {
.withPipelinesDirectoryOrFile(configFile)
.build();

System.out.println("Data Prepper Test with config file "+ configFile + " started at "+Instant.now());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please change this to an SLF4J log line.

LOG.info("PipelinesWithAcksIT with configu file {} started at {}", configFile, Instant.now());

dataPrepperTestRunner.start();
inMemorySourceAccessor = dataPrepperTestRunner.getInMemorySourceAccessor();
inMemorySinkAccessor = dataPrepperTestRunner.getInMemorySinkAccessor();
}

@AfterEach
void tearDown() {
System.out.println("Data Prepper Test stopped at "+Instant.now());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Again, use SLF4J.

dataPrepperTestRunner.stop();
}

Expand All @@ -60,7 +63,7 @@ void simple_pipeline_with_single_record() {
final int numRecords = 1;
inMemorySourceAccessor.submit(IN_MEMORY_IDENTIFIER, numRecords);

await().atMost(2000, TimeUnit.MILLISECONDS)
await().atMost(20000, TimeUnit.MILLISECONDS)
.untilAsserted(() -> {
List<Record<Event>> outputRecords = inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER);
assertThat(outputRecords, not(empty()));
Expand All @@ -76,7 +79,7 @@ void simple_pipeline_with_multiple_records() {
final int numRecords = 100;
inMemorySourceAccessor.submit(IN_MEMORY_IDENTIFIER, numRecords);

await().atMost(2000, TimeUnit.MILLISECONDS)
await().atMost(20000, TimeUnit.MILLISECONDS)
.untilAsserted(() -> {
List<Record<Event>> outputRecords = inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER);
assertThat(outputRecords, not(empty()));
Expand All @@ -91,7 +94,7 @@ void two_pipelines_with_multiple_records() {
final int numRecords = 100;
inMemorySourceAccessor.submit(IN_MEMORY_IDENTIFIER, numRecords);

await().atMost(5000, TimeUnit.MILLISECONDS)
await().atMost(20000, TimeUnit.MILLISECONDS)
.untilAsserted(() -> {
List<Record<Event>> outputRecords = inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER);
assertThat(outputRecords, not(empty()));
Expand Down
2 changes: 1 addition & 1 deletion ...pper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/InMemorySource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public void run() {
try {
final List<Record<Event>> records = inMemorySourceAccessor.read(testingKey);
if (records.size() == 0) {
Thread.sleep(1000);
Thread.sleep(50);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice!

continue;
}
AcknowledgementSet ackSet =
Expand Down
2 changes: 1 addition & 1 deletion ...es/org/opensearch/dataprepper/pipeline/acknowledgements/one-pipeline-ack-expiry-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
pipeline1:
pipeline-expiry-test:
delay: 2
source:
in_memory:
Expand Down
2 changes: 1 addition & 1 deletion ...ources/org/opensearch/dataprepper/pipeline/acknowledgements/one-pipeline-three-sinks.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
pipeline1:
pipeline-three-sinks-test:
delay: 2
source:
in_memory:
Expand Down
2 changes: 1 addition & 1 deletion ...ationTest/resources/org/opensearch/dataprepper/pipeline/acknowledgements/simple-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
simple-pipeline:
simple-pipeline-test:
delay: 10
source:
in_memory:
Expand Down
14 changes: 7 additions & 7 deletions ...urces/org/opensearch/dataprepper/pipeline/acknowledgements/three-pipeline-route-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
pipeline1:
three-pipelines-route-test-1:
delay: 2
source:
in_memory:
Expand All @@ -9,27 +9,27 @@ pipeline1:
- other_route: '/status >= 300 or /status < 200'
sink:
- pipeline:
name: "pipeline2"
name: "three-pipelines-route-test-2"
routes:
- 2xx_route
- pipeline:
name: "pipeline3"
name: "three-pipelines-route-test-3"
routes:
- other_route

pipeline2:
three-pipelines-route-test-2:
source:
pipeline:
name: "pipeline1"
name: "three-pipelines-route-test-1"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
acknowledgments: true

pipeline3:
three-pipelines-route-test-3:
source:
pipeline:
name: "pipeline1"
name: "three-pipelines-route-test-1"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
Expand Down
14 changes: 7 additions & 7 deletions ...org/opensearch/dataprepper/pipeline/acknowledgements/three-pipelines-test-multi-sink.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
pipeline1:
three-pipelines-multi-sink-1:
delay: 2
source:
in_memory:
Expand All @@ -9,23 +9,23 @@ pipeline1:
testing_key: PipelinesWithAcksIT
acknowledgments: true
- pipeline:
name: "pipeline2"
name: "three-pipelines-multi-sink-2"

pipeline2:
three-pipelines-multi-sink-2:
source:
pipeline:
name: "pipeline1"
name: "three-pipelines-multi-sink-1"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
acknowledgments: true
- pipeline:
name: "pipeline3"
name: "three-pipelines-multi-sink-3"

pipeline3:
three-pipelines-multi-sink-3:
source:
pipeline:
name: "pipeline2"
name: "three-pipelines-multi-sink-2"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
Expand Down
14 changes: 7 additions & 7 deletions .../resources/org/opensearch/dataprepper/pipeline/acknowledgements/three-pipelines-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
pipeline1:
three-pipelines-test-1:
delay: 2
source:
in_memory:
testing_key: PipelinesWithAcksIT
acknowledgments: true
sink:
- pipeline:
name: "pipeline2"
name: "three-pipelines-test-2"

pipeline2:
three-pipelines-test-2:
source:
pipeline:
name: "pipeline1"
name: "three-pipelines-test-1"
sink:
- pipeline:
name: "pipeline3"
name: "three-pipelines-test-3"

pipeline3:
three-pipelines-test-3:
source:
pipeline:
name: "pipeline2"
name: "three-pipelines-test-2"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
Expand Down
14 changes: 7 additions & 7 deletions ...ces/org/opensearch/dataprepper/pipeline/acknowledgements/two-parallel-pipelines-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
pipeline1:
two-parallel-pipelines-test-1:
delay: 2
source:
in_memory:
testing_key: PipelinesWithAcksIT
acknowledgments: true
sink:
- pipeline:
name: "pipeline2"
name: "two-parallel-pipelines-test-2"
- pipeline:
name: "pipeline3"
name: "two-parallel-pipelines-test-3"

pipeline2:
two-parallel-pipelines-test-2:
source:
pipeline:
name: "pipeline1"
name: "two-parallel-pipelines-test-1"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
acknowledgments: true

pipeline3:
two-parallel-pipelines-test-3:
source:
pipeline:
name: "pipeline1"
name: "two-parallel-pipelines-test-1"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
Expand Down
8 changes: 4 additions & 4 deletions ...st/resources/org/opensearch/dataprepper/pipeline/acknowledgements/two-pipelines-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
pipeline1:
two-pipelines-test-1:
delay: 2
source:
in_memory:
testing_key: PipelinesWithAcksIT
acknowledgments: true
sink:
- pipeline:
name: "pipeline2"
name: "two-pipelines-test-2"

pipeline2:
two-pipelines-test-2:
source:
pipeline:
name: "pipeline1"
name: "two-pipelines-test-1"
sink:
- in_memory:
testing_key: PipelinesWithAcksIT
Expand Down
Loading