From c439f52097840c5c277cc7cca96f6c35f8345678 Mon Sep 17 00:00:00 2001 From: Tomasz Konieczny Date: Fri, 8 Dec 2023 13:56:35 +0100 Subject: [PATCH] feat: executor tests - special cases added (#4743) * executor tests - special cases added * executor tests - jmeter - branch names fixed --- .../container-executor-large-artifacts.yaml | 79 +++++++++++++++++++ .../container-executor-large-logs.yaml | 55 +++++++++++++ test/jmeter/executor-tests/crd/other.yaml | 10 +-- test/scripts/executor-tests/run.sh | 36 +++++++-- .../edge-cases-expected-fails.yaml} | 0 test/special-cases/large-artifacts.yaml | 75 ++++++++++++++++++ test/special-cases/large-logs.yaml | 36 +++++++++ .../edge-cases-expected-fails.yaml} | 0 ...ted-failure-parallel-testsuite-delays.yaml | 23 ++++++ .../suites/special-cases/large-artifacts.yaml | 24 ++++++ test/suites/special-cases/large-logs.yaml | 21 +++++ .../special-cases/parallel-testsuite-2.yaml | 20 +++++ .../parallel-testsuite-delays.yaml | 23 ++++++ .../parallel-testsuite-downloadArtifacts.yaml | 23 ++++++ ...parallel-testsuite-downloadArtifacts2.yaml | 20 +++++ .../parallel-testsuite-large-logs.yaml | 30 +++++++ 16 files changed, 465 insertions(+), 10 deletions(-) create mode 100644 test/executors/container-executor-large-artifacts.yaml create mode 100644 test/executors/container-executor-large-logs.yaml rename test/{edge-cases/failures.yaml => special-cases/edge-cases-expected-fails.yaml} (100%) create mode 100644 test/special-cases/large-artifacts.yaml create mode 100644 test/special-cases/large-logs.yaml rename test/suites/{edge-cases-expected-failures.yaml => special-cases/edge-cases-expected-fails.yaml} (100%) create mode 100644 test/suites/special-cases/expected-failure-parallel-testsuite-delays.yaml create mode 100644 test/suites/special-cases/large-artifacts.yaml create mode 100644 test/suites/special-cases/large-logs.yaml create mode 100644 test/suites/special-cases/parallel-testsuite-2.yaml create mode 100644 test/suites/special-cases/parallel-testsuite-delays.yaml create mode 100644 test/suites/special-cases/parallel-testsuite-downloadArtifacts.yaml create mode 100644 test/suites/special-cases/parallel-testsuite-downloadArtifacts2.yaml create mode 100644 test/suites/special-cases/parallel-testsuite-large-logs.yaml diff --git a/test/executors/container-executor-large-artifacts.yaml b/test/executors/container-executor-large-artifacts.yaml new file mode 100644 index 00000000000..bebd4255bdf --- /dev/null +++ b/test/executors/container-executor-large-artifacts.yaml @@ -0,0 +1,79 @@ +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-1gb-artifact +spec: + types: + - container-executor-1gb-artifact/test + executor_type: container + image: alpine:3.17.2 + command: + - /bin/sh + - -c + - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/zero of=1g.img bs=1 count=0 seek=1G; ls -lah + features: + - artifacts +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-1mb-artifact +spec: + types: + - container-executor-1mb-artifact/test + executor_type: container + image: alpine:3.17.2 + command: + - /bin/sh + - -c + - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/zero of=1mb.img bs=1024 count=0 seek=1024; ls -lah + features: + - artifacts +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-1gb-urandom-artifact +spec: + types: + - container-executor-1gb-urandom-artifact/test + executor_type: container + image: alpine:3.17.2 + command: + - /bin/sh + - -c + - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/urandom of=1g.img bs=64M count=16; ls -lah + features: + - artifacts +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-1mb-urandom-artifact +spec: + types: + - container-executor-1mb-urandom-artifact/test + executor_type: container + image: alpine:3.17.2 + command: + - /bin/sh + - -c + - cd /share; mkdir artifacts; cd artifacts; dd if=/dev/urandom of=1mb.img bs=1024 count=0 seek=1024; ls -lah + features: + - artifacts +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-1gb-openssl-artifact +spec: + types: + - container-executor-1gb-openssl-artifact/test + executor_type: container + image: alpine/openssl:latest + command: + - /bin/sh + - -c + - cd /share && mkdir artifacts && cd artifacts && openssl rand -out 1g.txt -base64 $(( 2**30 * 3/4 )) ; ls -lah && echo "File generated - OK" + features: + - artifacts diff --git a/test/executors/container-executor-large-logs.yaml b/test/executors/container-executor-large-logs.yaml new file mode 100644 index 00000000000..db670c2458e --- /dev/null +++ b/test/executors/container-executor-large-logs.yaml @@ -0,0 +1,55 @@ +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-large-logs-1mb +spec: + types: + - container-executor-large-logs-1mb/test + executor_type: container + image: bash:5.1 + command: + - bash + - -c + - 'for iteration in {1..100}; do printf "\niteration $iteration\n"; for sets in {1..20}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..20}; printf "_LINE_END"; printf "\n"; done; done' # 100 iterations - about 1.1MB +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-large-logs-11mb +spec: + types: + - container-executor-large-logs-11mb/test + executor_type: container + image: bash:5.1 + command: + - bash + - -c + - 'for iteration in {1..1000}; do sleep 2; printf "\niteration $iteration\n"; for sets in {1..20}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..20}; printf "_LINE_END"; printf "\n"; done; done' # 1000 iterations - about 11MB +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-large-logs-22mb +spec: + types: + - container-executor-large-logs-22mb/test + executor_type: container + image: bash:5.1 + command: + - bash + - -c + - 'for iteration in {1..2000}; do sleep 1; printf "\niteration $iteration\n"; for sets in {1..20}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..20}; printf "_LINE_END"; printf "\n"; done; done' # 2000 iterations - about 22MB +--- +apiVersion: executor.testkube.io/v1 +kind: Executor +metadata: + name: container-executor-large-logs-long-lines +spec: + types: + - container-executor-large-logs-long-lines/test + executor_type: container + image: bash:5.1 + command: + - bash + - -c + - 'for iteration in {1..100}; do printf "\niteration $iteration\n"; for sets in {1..1}; do printf "LINE_BEGINNING_"; printf "abc DEFghi JKL ASDF BBB CCC%.0s" {1..3000}; printf "_LINE_END"; printf "\n"; done; done' # 100 lines with 81k characters each diff --git a/test/jmeter/executor-tests/crd/other.yaml b/test/jmeter/executor-tests/crd/other.yaml index dc3d68be3d3..845797cdf83 100644 --- a/test/jmeter/executor-tests/crd/other.yaml +++ b/test/jmeter/executor-tests/crd/other.yaml @@ -11,7 +11,7 @@ spec: repository: type: git uri: https://github.com/kubeshop/testkube.git - branch: executor-tests-jmeterd-extended # TODO: update before merge + branch: main path: test/jmeter/executor-tests/jmeter-executor-smoke-incorrect-url.jmx executionRequest: negativeTest: true @@ -31,7 +31,7 @@ spec: repository: type: git uri: https://github.com/kubeshop/testkube.git - branch: executor-tests-jmeterd-extended # TODO: update before merge + branch: main path: test/jmeter/executor-tests/jmeter-executor-smoke-incorrect-url.jmx executionRequest: negativeTest: true @@ -51,7 +51,7 @@ spec: repository: type: git uri: https://github.com/kubeshop/testkube.git - branch: executor-tests-jmeterd-extended # TODO: update before merge + branch: main path: test/jmeter/executor-tests/jmeter-executor-smoke-incorrect-url.jmx executionRequest: negativeTest: true @@ -84,7 +84,7 @@ spec: repository: type: git uri: https://github.com/kubeshop/testkube.git - branch: executor-tests-jmeterd-extended # TODO: update before merge + branch: main path: test/jmeter/executor-tests/jmeter-executor-smoke-correct-url-failed-assertion.jmx executionRequest: negativeTest: true @@ -104,7 +104,7 @@ spec: repository: type: git uri: https://github.com/kubeshop/testkube.git - branch: executor-tests-jmeterd-extended # TODO: update before merge + branch: main path: test/jmeter/executor-tests/jmeter-executor-smoke-correct-url-failed-assertion.jmx executionRequest: negativeTest: true diff --git a/test/scripts/executor-tests/run.sh b/test/scripts/executor-tests/run.sh index 2b4d7c436f2..137ce564a5e 100755 --- a/test/scripts/executor-tests/run.sh +++ b/test/scripts/executor-tests/run.sh @@ -287,15 +287,37 @@ soapui-smoke() { common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" } -edge-cases-failures() { - name="Edge Cases - Failures" - test_crd_file="test/edge-cases/failures.yaml" +special-cases-failures() { + name="Special Cases - Edge Cases - Expected Failures" + test_crd_file="test/special-cases/edge-cases-expected-fails.yaml" testsuite_name="expected-fail" - testsuite_file="test/suites/edge-cases-expected-failures.yaml" + testsuite_file="test/suites/special-cases/edge-cases-expected-fails.yaml" common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" } +special-cases-large-logs() { + name="Special Cases - Large logs" + test_crd_file="test/special-cases/large-logs.yaml" + testsuite_name="large-logs" + testsuite_file="test/suites/special-cases/large-logs.yaml" + + custom_executor_crd_file="test/executors/container-executor-large-logs.yaml" + + common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file" +} + +special-cases-large-artifacts() { + name="Special Cases - Large artifacts" + test_crd_file="test/special-cases/large-artifacts.yaml" + testsuite_name="large-artifacts" + testsuite_file="test/suites/special-cases/large-artifacts.yaml" + + custom_executor_crd_file="test/executors/container-executor-large-artifacts.yaml" + + common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file" +} + main() { case $executor_type in all) @@ -317,7 +339,6 @@ main() { postman-smoke playwright-smoke soapui-smoke - edge-cases-failures ;; smoke) artillery-smoke @@ -337,6 +358,11 @@ main() { postman-smoke soapui-smoke ;; + special) + special-cases-failures + special-cases-large-logs + special-cases-large-artifacts + ;; *) $executor_type ;; diff --git a/test/edge-cases/failures.yaml b/test/special-cases/edge-cases-expected-fails.yaml similarity index 100% rename from test/edge-cases/failures.yaml rename to test/special-cases/edge-cases-expected-fails.yaml diff --git a/test/special-cases/large-artifacts.yaml b/test/special-cases/large-artifacts.yaml new file mode 100644 index 00000000000..40ee52cfe94 --- /dev/null +++ b/test/special-cases/large-artifacts.yaml @@ -0,0 +1,75 @@ + +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-artifacts-1gb-test + labels: + core-tests: special-cases-large-artifacts +spec: + type: container-executor-1gb-artifact/test + executionRequest: + artifactRequest: + storageClassName: standard + volumeMountPath: /share + dirs: + - artifacts +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-artifacts-1mb-test + labels: + core-tests: special-cases-large-artifacts +spec: + type: container-executor-1mb-artifact/test + executionRequest: + artifactRequest: + storageClassName: standard + volumeMountPath: /share + dirs: + - artifacts +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-artifacts-1gb-urandom-test + labels: + core-tests: special-cases-large-artifacts +spec: + type: container-executor-1gb-urandom-artifact/test + executionRequest: + artifactRequest: + storageClassName: standard + volumeMountPath: /share + dirs: + - artifacts +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-artifacts-1mb-urandom-test + labels: + core-tests: special-cases-large-artifacts +spec: + type: container-executor-1mb-urandom-artifact/test + executionRequest: + artifactRequest: + storageClassName: standard + volumeMountPath: /share + dirs: + - artifacts +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-artifacts-1gb-openssl-test + labels: + core-tests: special-cases-large-artifacts +spec: + type: container-executor-1gb-openssl-artifact/test + executionRequest: + artifactRequest: + storageClassName: standard + volumeMountPath: /share + dirs: + - artifacts diff --git a/test/special-cases/large-logs.yaml b/test/special-cases/large-logs.yaml new file mode 100644 index 00000000000..6cdee9753db --- /dev/null +++ b/test/special-cases/large-logs.yaml @@ -0,0 +1,36 @@ + +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-logs-1mb-test + labels: + core-tests: special-cases-large-logs +spec: + type: container-executor-large-logs-1mb/test +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-logs-11mb-test + labels: + core-tests: special-cases-large-logs +spec: + type: container-executor-large-logs-11mb/test +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-logs-22mb-test + labels: + core-tests: special-cases-large-logs +spec: + type: container-executor-large-logs-22mb/test +--- +apiVersion: tests.testkube.io/v3 +kind: Test +metadata: + name: large-logs-long-lines-81k-test + labels: + core-tests: special-cases-large-logs +spec: + type: container-executor-large-logs-long-lines/test \ No newline at end of file diff --git a/test/suites/edge-cases-expected-failures.yaml b/test/suites/special-cases/edge-cases-expected-fails.yaml similarity index 100% rename from test/suites/edge-cases-expected-failures.yaml rename to test/suites/special-cases/edge-cases-expected-fails.yaml diff --git a/test/suites/special-cases/expected-failure-parallel-testsuite-delays.yaml b/test/suites/special-cases/expected-failure-parallel-testsuite-delays.yaml new file mode 100644 index 00000000000..1dc2ecf111b --- /dev/null +++ b/test/suites/special-cases/expected-failure-parallel-testsuite-delays.yaml @@ -0,0 +1,23 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: expected-failure-parallel-testsuite-delays + labels: + core-tests: special-cases-expected-failure +spec: + steps: + - stopOnFailure: false + execute: + - test: k6-executor-smoke + - delay: 30s + - test: container-executor-curl-smoke + - stopOnFailure: false + execute: + - test: k6-executor-smoke + - test: postman-executor-smoke + - delay: 12s + - stopOnFailure: false + execute: + - test: soapui-executor-smoke + - delay: 2s + - delay: 1.223s \ No newline at end of file diff --git a/test/suites/special-cases/large-artifacts.yaml b/test/suites/special-cases/large-artifacts.yaml new file mode 100644 index 00000000000..571c3dc9d9e --- /dev/null +++ b/test/suites/special-cases/large-artifacts.yaml @@ -0,0 +1,24 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: large-artifacts + labels: + core-tests: special-cases-large-artifacts +spec: + description: "Special cases - large artifacts" + steps: + - stopOnFailure: false + execute: + - test: large-artifacts-1gb-test + - stopOnFailure: false + execute: + - test: large-artifacts-1mb-test + - stopOnFailure: false + execute: + - test: large-artifacts-1gb-urandom-test + - stopOnFailure: false + execute: + - test: large-artifacts-1mb-urandom-test + - stopOnFailure: false + execute: + - test: large-artifacts-1gb-openssl-test \ No newline at end of file diff --git a/test/suites/special-cases/large-logs.yaml b/test/suites/special-cases/large-logs.yaml new file mode 100644 index 00000000000..dd8f667e6be --- /dev/null +++ b/test/suites/special-cases/large-logs.yaml @@ -0,0 +1,21 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: large-logs + labels: + core-tests: special-cases-large-logs +spec: + description: "Special cases - large logs" + steps: + - stopOnFailure: false + execute: + - test: large-logs-1mb-test + - stopOnFailure: false + execute: + - test: large-logs-11mb-test + - stopOnFailure: false + execute: + - test: large-logs-22mb-test + - stopOnFailure: false + execute: + - test: large-logs-long-lines-81k-test \ No newline at end of file diff --git a/test/suites/special-cases/parallel-testsuite-2.yaml b/test/suites/special-cases/parallel-testsuite-2.yaml new file mode 100644 index 00000000000..f19ff0dde64 --- /dev/null +++ b/test/suites/special-cases/parallel-testsuite-2.yaml @@ -0,0 +1,20 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: parallel-testsuite-2 + labels: + core-tests: special-cases +spec: + description: "parallel testsuite 2" + steps: + - stopOnFailure: false + execute: + - test: artillery-executor-smoke + - test: k6-executor-smoke + - stopOnFailure: false + execute: + - test: k6-executor-smoke-git-file + - stopOnFailure: false + execute: + - test: artillery-executor-smoke-negative + - test: container-executor-curl-smoke diff --git a/test/suites/special-cases/parallel-testsuite-delays.yaml b/test/suites/special-cases/parallel-testsuite-delays.yaml new file mode 100644 index 00000000000..b6d9fb26779 --- /dev/null +++ b/test/suites/special-cases/parallel-testsuite-delays.yaml @@ -0,0 +1,23 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: parallel-testsuite-delays + labels: + core-tests: special-cases +spec: + steps: + - stopOnFailure: false + execute: + - test: k6-executor-smoke + - test: container-executor-curl-smoke + - delay: 10s + - stopOnFailure: false + execute: + - test: k6-executor-smoke + - test: postman-executor-smoke + - delay: 12s + - stopOnFailure: false + execute: + - test: soapui-executor-smoke + - delay: 2s + - delay: 1.223s \ No newline at end of file diff --git a/test/suites/special-cases/parallel-testsuite-downloadArtifacts.yaml b/test/suites/special-cases/parallel-testsuite-downloadArtifacts.yaml new file mode 100644 index 00000000000..629d876b8bc --- /dev/null +++ b/test/suites/special-cases/parallel-testsuite-downloadArtifacts.yaml @@ -0,0 +1,23 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: parallel-testsuite-downloadartifacts + labels: + core-tests: special-cases +spec: + steps: + - stopOnFailure: false + execute: + - test: artillery-executor-smoke + - test: soapui-executor-smoke + - stopOnFailure: false + execute: + - test: container-executor-curl-smoke + - test: container-executor-curl-smoke + - test: soapui-executor-smoke + - stopOnFailure: false + downloadArtifacts: + previousStepNumbers: [] + execute: + - test: expected-fail-container-executor-k6-wrong-dir + - test: soapui-executor-smoke \ No newline at end of file diff --git a/test/suites/special-cases/parallel-testsuite-downloadArtifacts2.yaml b/test/suites/special-cases/parallel-testsuite-downloadArtifacts2.yaml new file mode 100644 index 00000000000..cda6fd1c9fc --- /dev/null +++ b/test/suites/special-cases/parallel-testsuite-downloadArtifacts2.yaml @@ -0,0 +1,20 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: parallel-testsuite-downloadartifacts2 + labels: + core-tests: special-cases +spec: + description: "standard" + steps: + - stopOnFailure: false + execute: + - test: artillery-executor-smoke + - stopOnFailure: false + execute: + - test: container-executor-curl-smoke + - stopOnFailure: false + downloadArtifacts: + allPreviousSteps: true + execute: + - test: container-executor-curl-smoke \ No newline at end of file diff --git a/test/suites/special-cases/parallel-testsuite-large-logs.yaml b/test/suites/special-cases/parallel-testsuite-large-logs.yaml new file mode 100644 index 00000000000..cb97a2478d9 --- /dev/null +++ b/test/suites/special-cases/parallel-testsuite-large-logs.yaml @@ -0,0 +1,30 @@ +apiVersion: tests.testkube.io/v3 +kind: TestSuite +metadata: + name: parallel-large-logs + labels: + core-tests: special-cases +spec: + steps: + - stopOnFailure: false + execute: + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-11mb-test + - test: large-logs-22mb-test + - test: large-logs-22mb-test + - test: large-logs-22mb-test + - test: large-logs-22mb-test \ No newline at end of file