feat: executor tests - special cases added (#4743)

* executor tests - special cases added * executor tests - jmeter - branch names fixed
kubeshop · Dec 8, 2023 · c439f52 · c439f52
1 parent 6d8a000
commit c439f52
Show file tree

Hide file tree

Showing 16 changed files with 465 additions and 10 deletions.
diff --git a/test/executors/container-executor-large-artifacts.yaml b/test/executors/container-executor-large-artifacts.yaml
@@ -0,0 +1,79 @@
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-1gb-artifact
+spec:
+  types:
+    - container-executor-1gb-artifact/test
+  executor_type: container
+  image: alpine:3.17.2
+  command:
+    - /bin/sh
+    - -c
+    - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/zero of=1g.img bs=1 count=0 seek=1G; ls -lah
+  features:
+    - artifacts
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-1mb-artifact
+spec:
+  types:
+    - container-executor-1mb-artifact/test
+  executor_type: container
+  image: alpine:3.17.2
+  command:
+    - /bin/sh
+    - -c
+    - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/zero of=1mb.img bs=1024 count=0 seek=1024; ls -lah
+  features:
+    - artifacts
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-1gb-urandom-artifact
+spec:
+  types:
+    - container-executor-1gb-urandom-artifact/test
+  executor_type: container
+  image: alpine:3.17.2
+  command:
+    - /bin/sh
+    - -c
+    - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/urandom of=1g.img bs=64M count=16; ls -lah
+  features:
+    - artifacts
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-1mb-urandom-artifact
+spec:
+  types:
+    - container-executor-1mb-urandom-artifact/test
+  executor_type: container
+  image: alpine:3.17.2
+  command:
+    - /bin/sh
+    - -c
+    - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/urandom of=1mb.img bs=1024 count=0 seek=1024; ls -lah
+  features:
+    - artifacts
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-1gb-openssl-artifact
+spec:
+  types:
+    - container-executor-1gb-openssl-artifact/test
+  executor_type: container
+  image: alpine/openssl:latest
+  command:
+    - /bin/sh
+    - -c
+    - cd /share && mkdir artifacts && cd artifacts && openssl rand -out 1g.txt -base64 $(( 2**30 * 3/4 )) ; ls -lah && echo "File generated - OK"
+  features:
+    - artifacts
diff --git a/test/executors/container-executor-large-logs.yaml b/test/executors/container-executor-large-logs.yaml
@@ -0,0 +1,55 @@
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-large-logs-1mb
+spec:
+  types:
+    - container-executor-large-logs-1mb/test
+  executor_type: container
+  image: bash:5.1
+  command:
+    - bash
+    - -c
+    - 'for iteration in {1..100}; do printf "\niteration $iteration\n"; for sets in {1..20}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..20}; printf "_LINE_END"; printf "\n"; done; done' # 100 iterations - about 1.1MB
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-large-logs-11mb
+spec:
+  types:
+    - container-executor-large-logs-11mb/test
+  executor_type: container
+  image: bash:5.1
+  command:
+    - bash
+    - -c
+    - 'for iteration in {1..1000}; do sleep 2; printf "\niteration $iteration\n"; for sets in {1..20}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..20}; printf "_LINE_END"; printf "\n"; done; done' # 1000 iterations - about 11MB
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-large-logs-22mb
+spec:
+  types:
+    - container-executor-large-logs-22mb/test
+  executor_type: container
+  image: bash:5.1
+  command:
+    - bash
+    - -c
+    - 'for iteration in {1..2000}; do sleep 1; printf "\niteration $iteration\n"; for sets in {1..20}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..20}; printf "_LINE_END"; printf "\n"; done; done' # 2000 iterations - about 22MB
+---
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-large-logs-long-lines
+spec:
+  types:
+    - container-executor-large-logs-long-lines/test
+  executor_type: container
+  image: bash:5.1
+  command:
+    - bash
+    - -c
+    - 'for iteration in {1..100}; do printf "\niteration $iteration\n"; for sets in {1..1}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..3000}; printf "_LINE_END"; printf "\n"; done; done' # 100 lines with 81k characters each
diff --git a/test/jmeter/executor-tests/crd/other.yaml b/test/jmeter/executor-tests/crd/other.yaml
@@ -11,7 +11,7 @@ spec:
     repository:
       type: git
       uri: https://github.com/kubeshop/testkube.git
-      branch: executor-tests-jmeterd-extended # TODO: update before merge
+      branch: main
       path: test/jmeter/executor-tests/jmeter-executor-smoke-incorrect-url.jmx
   executionRequest:
     negativeTest: true
@@ -31,7 +31,7 @@ spec:
     repository:
       type: git
       uri: https://github.com/kubeshop/testkube.git
-      branch: executor-tests-jmeterd-extended # TODO: update before merge
+      branch: main
       path: test/jmeter/executor-tests/jmeter-executor-smoke-incorrect-url.jmx
   executionRequest:
     negativeTest: true
@@ -51,7 +51,7 @@ spec:
     repository:
       type: git
       uri: https://github.com/kubeshop/testkube.git
-      branch: executor-tests-jmeterd-extended # TODO: update before merge
+      branch: main
       path: test/jmeter/executor-tests/jmeter-executor-smoke-incorrect-url.jmx
   executionRequest:
     negativeTest: true
@@ -84,7 +84,7 @@ spec:
     repository:
       type: git
       uri: https://github.com/kubeshop/testkube.git
-      branch: executor-tests-jmeterd-extended # TODO: update before merge
+      branch: main
       path: test/jmeter/executor-tests/jmeter-executor-smoke-correct-url-failed-assertion.jmx
   executionRequest:
     negativeTest: true
@@ -104,7 +104,7 @@ spec:
     repository:
       type: git
       uri: https://github.com/kubeshop/testkube.git
-      branch: executor-tests-jmeterd-extended # TODO: update before merge
+      branch: main
       path: test/jmeter/executor-tests/jmeter-executor-smoke-correct-url-failed-assertion.jmx
   executionRequest:
     negativeTest: true

diff --git a/test/scripts/executor-tests/run.sh b/test/scripts/executor-tests/run.sh
@@ -287,15 +287,37 @@ soapui-smoke() {
   common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file"
 }
 
-edge-cases-failures() {
-  name="Edge Cases - Failures"
-  test_crd_file="test/edge-cases/failures.yaml"
+special-cases-failures() {
+  name="Special Cases - Edge Cases - Expected Failures"
+  test_crd_file="test/special-cases/edge-cases-expected-fails.yaml"
   testsuite_name="expected-fail"
-  testsuite_file="test/suites/edge-cases-expected-failures.yaml"
+  testsuite_file="test/suites/special-cases/edge-cases-expected-fails.yaml"
 
   common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file"
 }
 
+special-cases-large-logs() {
+  name="Special Cases - Large logs"
+  test_crd_file="test/special-cases/large-logs.yaml"
+  testsuite_name="large-logs"
+  testsuite_file="test/suites/special-cases/large-logs.yaml"
+
+  custom_executor_crd_file="test/executors/container-executor-large-logs.yaml"
+
+  common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file"
+}
+
+special-cases-large-artifacts() {
+  name="Special Cases - Large artifacts"
+  test_crd_file="test/special-cases/large-artifacts.yaml"
+  testsuite_name="large-artifacts"
+  testsuite_file="test/suites/special-cases/large-artifacts.yaml"
+
+  custom_executor_crd_file="test/executors/container-executor-large-artifacts.yaml"
+
+  common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file"
+}
+
 main() {
   case $executor_type in
     all)
@@ -317,7 +339,6 @@ main() {
       postman-smoke
       playwright-smoke
       soapui-smoke
-      edge-cases-failures
       ;;
     smoke)
       artillery-smoke
@@ -337,6 +358,11 @@ main() {
       postman-smoke
       soapui-smoke
       ;;
+    special)
+      special-cases-failures
+      special-cases-large-logs
+      special-cases-large-artifacts
+      ;;
     *)
       $executor_type
       ;;

diff --git a/test/edge-cases/failures.yaml → ...cial-cases/edge-cases-expected-fails.yaml b/test/edge-cases/failures.yaml → ...cial-cases/edge-cases-expected-fails.yaml
diff --git a/test/special-cases/large-artifacts.yaml b/test/special-cases/large-artifacts.yaml
@@ -0,0 +1,75 @@
+
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-artifacts-1gb-test
+  labels:
+    core-tests: special-cases-large-artifacts
+spec:
+  type: container-executor-1gb-artifact/test
+  executionRequest:
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /share
+      dirs:
+        - artifacts
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-artifacts-1mb-test
+  labels:
+    core-tests: special-cases-large-artifacts
+spec:
+  type: container-executor-1mb-artifact/test
+  executionRequest:
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /share
+      dirs:
+        - artifacts
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-artifacts-1gb-urandom-test
+  labels:
+    core-tests: special-cases-large-artifacts
+spec:
+  type: container-executor-1gb-urandom-artifact/test
+  executionRequest:
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /share
+      dirs:
+        - artifacts
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-artifacts-1mb-urandom-test
+  labels:
+    core-tests: special-cases-large-artifacts
+spec:
+  type: container-executor-1mb-urandom-artifact/test
+  executionRequest:
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /share
+      dirs:
+        - artifacts
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-artifacts-1gb-openssl-test
+  labels:
+    core-tests: special-cases-large-artifacts
+spec:
+  type: container-executor-1gb-openssl-artifact/test
+  executionRequest:
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /share
+      dirs:
+        - artifacts
diff --git a/test/special-cases/large-logs.yaml b/test/special-cases/large-logs.yaml
@@ -0,0 +1,36 @@
+
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-logs-1mb-test
+  labels:
+    core-tests: special-cases-large-logs
+spec:
+  type: container-executor-large-logs-1mb/test
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-logs-11mb-test
+  labels:
+    core-tests: special-cases-large-logs
+spec:
+  type: container-executor-large-logs-11mb/test
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-logs-22mb-test
+  labels:
+    core-tests: special-cases-large-logs
+spec:
+  type: container-executor-large-logs-22mb/test
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: large-logs-long-lines-81k-test
+  labels:
+    core-tests: special-cases-large-logs
+spec:
+  type: container-executor-large-logs-long-lines/test
diff --git a/.../suites/edge-cases-expected-failures.yaml → ...cial-cases/edge-cases-expected-fails.yaml b/.../suites/edge-cases-expected-failures.yaml → ...cial-cases/edge-cases-expected-fails.yaml
diff --git a/test/suites/special-cases/expected-failure-parallel-testsuite-delays.yaml b/test/suites/special-cases/expected-failure-parallel-testsuite-delays.yaml
@@ -0,0 +1,23 @@
+apiVersion: tests.testkube.io/v3
+kind: TestSuite
+metadata:
+  name: expected-failure-parallel-testsuite-delays
+  labels:
+    core-tests: special-cases-expected-failure
+spec:
+  steps:
+  - stopOnFailure: false
+    execute:
+    - test: k6-executor-smoke
+    - delay: 30s
+    - test: container-executor-curl-smoke
+  - stopOnFailure: false
+    execute:
+    - test: k6-executor-smoke
+    - test: postman-executor-smoke
+    - delay: 12s
+  - stopOnFailure: false
+    execute:
+    - test: soapui-executor-smoke
+    - delay: 2s
+    - delay: 1.223s