fix(kafka-runner): handle flow with multiple trigger that trigger mul…

…tiple times next flow
kestra-io · Feb 23, 2022 · 7120872 · 7120872
1 parent 53d2f1b
commit 7120872
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 6 deletions.
diff --git a/core/src/main/java/io/kestra/core/models/executions/statistics/DailyExecutionStatistics.java b/core/src/main/java/io/kestra/core/models/executions/statistics/DailyExecutionStatistics.java
@@ -30,7 +30,8 @@ public class DailyExecutionStatistics {
         State.Type.SUCCESS, 0L,
         State.Type.WARNING, 0L,
         State.Type.FAILED, 0L,
-        State.Type.KILLED, 0L
+        State.Type.KILLED, 0L,
+        State.Type.PAUSED, 0L
     ));
 
     @Value

diff --git a/core/src/test/resources/flows/valids/trigger-flow-listener-no-inputs.yaml b/core/src/test/resources/flows/valids/trigger-flow-listener-no-inputs.yaml
@@ -17,3 +17,12 @@ triggers:
       - type: io.kestra.core.models.conditions.types.ExecutionFlowCondition
         namespace: io.kestra.tests
         flowId: trigger-flow
+  - id: listen-flow-failed
+    type: io.kestra.core.models.triggers.types.Flow
+    conditions:
+      - type: io.kestra.core.models.conditions.types.ExecutionStatusCondition
+        in:
+          - FAILED
+      - type: io.kestra.core.models.conditions.types.ExecutionFlowCondition
+        namespace: io.kestra.tests
+        flowId: other-flow
diff --git a/...rc/test/java/io/kestra/repository/elasticsearch/ElasticSearchExecutionRepositoryTest.java b/...rc/test/java/io/kestra/repository/elasticsearch/ElasticSearchExecutionRepositoryTest.java
@@ -171,14 +171,14 @@ void dailyGroupByFlowStatistics() {
         DailyExecutionStatistics second = result.get("io.kestra.unittest").get("second").get(10);
 
         assertThat(full.getDuration().getAvg().getSeconds(), greaterThan(0L));
-        assertThat(full.getExecutionCounts().size(), is(8));
+        assertThat(full.getExecutionCounts().size(), is(9));
         assertThat(full.getExecutionCounts().get(State.Type.FAILED), is(3L));
         assertThat(full.getExecutionCounts().get(State.Type.RUNNING), is(5L));
         assertThat(full.getExecutionCounts().get(State.Type.SUCCESS), is(7L));
         assertThat(full.getExecutionCounts().get(State.Type.CREATED), is(0L));
 
         assertThat(second.getDuration().getAvg().getSeconds(), greaterThan(0L));
-        assertThat(second.getExecutionCounts().size(), is(8));
+        assertThat(second.getExecutionCounts().size(), is(9));
         assertThat(second.getExecutionCounts().get(State.Type.SUCCESS), is(13L));
         assertThat(second.getExecutionCounts().get(State.Type.CREATED), is(0L));
 
@@ -193,7 +193,7 @@ void dailyGroupByFlowStatistics() {
         assertThat(result.get("io.kestra.unittest").size(), is(1));
         full = result.get("io.kestra.unittest").get("*").get(10);
         assertThat(full.getDuration().getAvg().getSeconds(), greaterThan(0L));
-        assertThat(full.getExecutionCounts().size(), is(8));
+        assertThat(full.getExecutionCounts().size(), is(9));
         assertThat(full.getExecutionCounts().get(State.Type.FAILED), is(3L));
         assertThat(full.getExecutionCounts().get(State.Type.RUNNING), is(5L));
         assertThat(full.getExecutionCounts().get(State.Type.SUCCESS), is(20L));
@@ -217,7 +217,7 @@ void dailyStatistics() {
         );
 
         assertThat(result.size(), is(11));
-        assertThat(result.get(10).getExecutionCounts().size(), is(8));
+        assertThat(result.get(10).getExecutionCounts().size(), is(9));
         assertThat(result.get(10).getDuration().getAvg().getSeconds(), greaterThan(0L));
 
         assertThat(result.get(10).getExecutionCounts().get(State.Type.FAILED), is(3L));
@@ -243,7 +243,7 @@ void taskRunsDailyStatistics() {
         );
 
         assertThat(result.size(), is(11));
-        assertThat(result.get(10).getExecutionCounts().size(), is(8));
+        assertThat(result.get(10).getExecutionCounts().size(), is(9));
         assertThat(result.get(10).getDuration().getAvg().getSeconds(), greaterThan(0L));
 
         assertThat(result.get(10).getExecutionCounts().get(State.Type.FAILED), is(3L * 2));

diff --git a/runner-kafka/src/main/java/io/kestra/runner/kafka/streams/FlowWithTriggerTransformer.java b/runner-kafka/src/main/java/io/kestra/runner/kafka/streams/FlowWithTriggerTransformer.java
@@ -27,7 +27,11 @@ public void init(ProcessorContext context) {
 
     @Override
     public Iterable<KeyValue<String, ExecutorFlowTrigger>> transform(String key, Executor value) {
+        // flowWithFlowTrigger return 1 result per flow per trigger but since we analysed the whole flow on FlowTrigger
+        // we deduplicate by flow
         return flowService.flowWithFlowTrigger(kafkaFlowExecutor.allLastVersion().stream())
+            .stream()
+            .collect(Collectors.toMap(o -> o.getFlow().uidWithoutRevision(), p -> p, (p, q) -> p)).values()
             .stream()
             .map(f -> KeyValue.pair(f.getFlow().uidWithoutRevision(), new ExecutorFlowTrigger(f.getFlow(), value.getExecution())))
             .collect(Collectors.toList());

diff --git a/runner-kafka/src/test/java/io/kestra/runner/kafka/KafkaExecutorTest.java b/runner-kafka/src/test/java/io/kestra/runner/kafka/KafkaExecutorTest.java
@@ -492,13 +492,17 @@ void flowTrigger() {
         io.kestra.runner.kafka.streams.ExecutorFlowTrigger executorFlowTrigger = executorFlowTriggerOutput().readRecord().value();
         assertThat(executorFlowTrigger.getFlowHavingTrigger().getId(), is("trigger-flow-listener-no-inputs"));
 
+        assertThat(executorFlowTriggerOutput().isEmpty(), is(true));
+
         startStream(this.executorFlowTrigger);
 
         executorFlowTriggerInput().pipeInput(executorFlowTrigger.getFlowHavingTrigger().uid(), executorFlowTrigger);
 
         Execution triggerExecution = executionOutput().readRecord().getValue();
         assertThat(triggerExecution.getState().getCurrent(), is(State.Type.CREATED));
         assertThat(triggerExecution.getFlowId(), is("trigger-flow-listener-no-inputs"));
+
+        assertThat(executionOutput().isEmpty(), is(true));
     }
 
     @Test