Vs markers (#4793)

split test matrix for vs and policies to reduce pipeline duration
nginxinc · Dec 14, 2023 · 4a9832a · 4a9832a
1 parent b84e64f
commit 4a9832a
Show file tree

Hide file tree

Showing 23 changed files with 51 additions and 6 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -256,20 +256,27 @@ jobs:
             echo "matrix={\"images\": \
                                                 [{\"image\": \"debian\", \"marker\": \"ingresses\"}, \
                                                 {\"image\": \"alpine\", \"marker\":\"vsr\"}, \
-                                                {\"image\": \"alpine\", \"marker\":\"policies\"}, \
-                                                {\"image\": \"debian\", \"marker\": \"vs\"}, \
+                                                {\"image\": \"alpine\", \"marker\":\"'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'\"}, \
+                                                {\"image\": \"alpine\", \"marker\":\"'policies_rl or policies_ac or policies_jwt or policies_mtls'\"}, \
+                                                {\"image\": \"debian\", \"marker\": \"'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'\"}, \
+                                                {\"image\": \"debian\", \"marker\": \"'vs_ipv6 or vs_rewrite or vs_responses or vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_certmanager'\"}, \
+                                                {\"image\": \"debian\", \"marker\": \"'vs_certmanager'\"}, \
                                                 {\"image\": \"ubi\", \"marker\": \"ts\"}, \
-                                                {\"image\": \"debian-plus\", \"marker\": \"vs\"}, \
+                                                {\"image\": \"debian-plus\", \"marker\": \"'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'\"}, \
+                                                {\"image\": \"debian-plus\", \"marker\": \"'vs_ipv6 or vs_rewrite or vs_responses or vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_certmanager'\"}, \
+                                                {\"image\": \"debian-plus\", \"marker\": \"vs_certmanager\"}, \
                                                 {\"image\": \"debian-plus\", \"marker\": \"ts\"}, \
                                                 {\"image\": \"alpine-plus\", \"marker\":\"ingresses\"}, \
                                                 {\"image\": \"alpine-plus\", \"marker\": \"vsr\"}, \
-                                                {\"image\": \"ubi-plus\", \"marker\": \"policies\"}, \
+                                                {\"image\": \"ubi-plus\", \"marker\":\"'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'\"}, \
+                                                {\"image\": \"ubi-plus\", \"marker\":\"'policies_rl or policies_ac or policies_jwt or policies_mtls'\"}, \
                                                 {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_integration\"}, \
                                                 {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_allow\"}, \
                                                 {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_waf_policies and not appprotect_waf_policies_allow'\"}, \
                                                 {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_grpc\"}, \
-                                                {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_watch or appprotect_batch'\"}, \
-                                                {\"image\": \"debian-plus-nap\", \"marker\": \"dos\"}], \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_watch or appprotect_batch or appprotect_integration'\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"'dos and not dos_learning'\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"dos_learning\"}], \
                                               \"k8s\": [\"${{ needs.checks.outputs.k8s_latest }}\"]}" >> $GITHUB_OUTPUT
           else
             echo "matrix={\"k8s\": [\"1.22.17\", \"1.23.17\", \"1.24.15\", \"1.25.11\", \"1.26.6\", \"1.27.3\", \"${{ needs.checks.outputs.k8s_latest }}\"], \

diff --git a/pyproject.toml b/pyproject.toml
@@ -32,13 +32,26 @@ markers =[
     "appprotect_watch",
     "appprotect_batch",
     "dos",
+    "dos_learning",
     "ingresses",
     "policies",
+    "policies_rl",
+    "policies_jwt",
+    "policies_ac",
+    "policies_mtls",
     "rewrite",
     "skip_for_nginx_oss",
     "smoke",
     "ts",
     "vs",
+    "vs_ipv6",
+    "vs_rewrite",
+    "vs_responses",
+    "vs_grpc",
+    "vs_redirects",
+    "vs_externaldns",
+    "vs_externalname",
+    "vs_certmanager",
     "vsr",
 ]
 testpaths = [

diff --git a/tests/suite/test_ac_policies.py b/tests/suite/test_ac_policies.py
@@ -66,6 +66,7 @@ def fin():
 
 
 @pytest.mark.policies
+@pytest.mark.policies_ac
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_ac_policies_vsr.py b/tests/suite/test_ac_policies_vsr.py
@@ -50,6 +50,7 @@ def fin():
 
 
 @pytest.mark.policies
+@pytest.mark.policies_ac
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

diff --git a/tests/suite/test_disable_ipv6.py b/tests/suite/test_disable_ipv6.py
@@ -21,6 +21,7 @@
 
 @pytest.mark.vs
 @pytest.mark.ts
+@pytest.mark.vs_ipv6
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup, transport_server_setup",
     [

diff --git a/tests/suite/test_dos.py b/tests/suite/test_dos.py
@@ -253,6 +253,7 @@ def test_dos_sec_logs_on(
         assert f'vs_name="{test_namespace}/dos-protected/name"' in log_contents
         assert "bad_actor" in log_contents
 
+    @pytest.mark.dos_learning
     def test_dos_under_attack_with_learning(
         self, kube_apis, ingress_controller_prerequisites, crd_ingress_controller_with_dos, dos_setup, test_namespace
     ):

diff --git a/tests/suite/test_egress_mtls.py b/tests/suite/test_egress_mtls.py
@@ -50,6 +50,7 @@ def teardown_policy(kube_apis, test_namespace, tls_secret, pol_name, mtls_secret
 
 
 @pytest.mark.policies
+@pytest.mark.policies_mtls
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_ingress_mtls.py b/tests/suite/test_ingress_mtls.py
@@ -63,6 +63,7 @@ def teardown_policy(kube_apis, test_namespace, tls_secret, pol_name, mtls_secret
 
 
 @pytest.mark.policies
+@pytest.mark.policies_mtls
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [
@@ -417,6 +418,7 @@ def test_ingress_mtls_policy_cert_crl(
 
 
 @pytest.mark.policies
+@pytest.mark.policies_mtls
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

diff --git a/tests/suite/test_jwt_policies.py b/tests/suite/test_jwt_policies.py
@@ -27,6 +27,7 @@
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.policies
+@pytest.mark.policies_jwt
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_jwt_policies_vsr.py b/tests/suite/test_jwt_policies_vsr.py
@@ -27,6 +27,7 @@
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.policies
+@pytest.mark.policies_jwt
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

diff --git a/tests/suite/test_rewrites.py b/tests/suite/test_rewrites.py
@@ -100,6 +100,7 @@ def fin():
 @pytest.mark.parametrize("crd_ingress_controller", [({"type": "complete"})], indirect=True)
 class TestRewrites:
     @pytest.mark.vs
+    @pytest.mark.vs_rewrite
     @pytest.mark.parametrize("path,args,cookies,expected", test_data)
     def test_vs_rewrite(self, vs_rewrites_setup, path, args, cookies, expected):
         """

diff --git a/tests/suite/test_rl_policies.py b/tests/suite/test_rl_policies.py
@@ -25,6 +25,7 @@
 
 
 @pytest.mark.policies
+@pytest.mark.policies_rl
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_rl_policies_vsr.py b/tests/suite/test_rl_policies_vsr.py
@@ -21,6 +21,7 @@
 
 
 @pytest.mark.policies
+@pytest.mark.policies_rl
 @pytest.mark.parametrize(
     "crd_ingress_controller, v_s_route_setup",
     [

diff --git a/tests/suite/test_virtual_server.py b/tests/suite/test_virtual_server.py
@@ -21,6 +21,7 @@
 
 
 @pytest.mark.vs
+@pytest.mark.vs_responses
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",

diff --git a/tests/suite/test_virtual_server_canned_responses.py b/tests/suite/test_virtual_server_canned_responses.py
@@ -15,6 +15,7 @@
 
 
 @pytest.mark.vs
+@pytest.mark.vs_responses
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_virtual_server_certmanager.py b/tests/suite/test_virtual_server_certmanager.py
@@ -12,6 +12,7 @@
 
 
 @pytest.mark.vs
+@pytest.mark.vs_certmanager
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller, create_certmanager, virtual_server_setup",
@@ -42,6 +43,7 @@ def test_responses_after_setup(self, kube_apis, crd_ingress_controller, create_c
 
 
 @pytest.mark.vs
+@pytest.mark.vs_certmanager
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller, create_certmanager, virtual_server_setup",
@@ -102,6 +104,7 @@ def test_virtual_server_no_cm(self, kube_apis, crd_ingress_controller, create_ce
 
 
 @pytest.mark.vs
+@pytest.mark.vs_certmanager
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller, create_certmanager, virtual_server_setup",

diff --git a/tests/suite/test_virtual_server_external_name.py b/tests/suite/test_virtual_server_external_name.py
@@ -85,6 +85,7 @@ def fin():
 
 
 @pytest.mark.vs
+@pytest.mark.vs_externalname
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",

diff --git a/tests/suite/test_virtual_server_externaldns.py b/tests/suite/test_virtual_server_externaldns.py
@@ -10,6 +10,7 @@
 
 
 @pytest.mark.vs
+@pytest.mark.vs_externaldns
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ed, create_externaldns, virtual_server_setup",

diff --git a/tests/suite/test_virtual_server_grpc.py b/tests/suite/test_virtual_server_grpc.py
@@ -80,6 +80,7 @@ def fin():
 
 
 @pytest.mark.vs
+@pytest.mark.vs_grpc
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
@@ -265,6 +266,7 @@ def test_config_after_enable_tls(
 
 
 @pytest.mark.vs
+@pytest.mark.vs_gprc
 @pytest.mark.smoke
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.parametrize(

diff --git a/tests/suite/test_virtual_server_mixed_grpc.py b/tests/suite/test_virtual_server_mixed_grpc.py
@@ -75,6 +75,7 @@ def fin():
 
 
 @pytest.mark.vs
+@pytest.mark.vs_grpc
 @pytest.mark.smoke
 @pytest.mark.flaky(max_runs=3)
 @pytest.mark.parametrize(

diff --git a/tests/suite/test_virtual_server_redirects.py b/tests/suite/test_virtual_server_redirects.py
@@ -13,6 +13,7 @@
 
 
 @pytest.mark.vs
+@pytest.mark.vs_redirects
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_virtual_server_tls_redirect.py b/tests/suite/test_virtual_server_tls_redirect.py
@@ -7,6 +7,7 @@
 
 
 @pytest.mark.vs
+@pytest.mark.vs_redirects
 @pytest.mark.parametrize(
     "crd_ingress_controller, virtual_server_setup",
     [

diff --git a/tests/suite/test_watch_namespace_label.py b/tests/suite/test_watch_namespace_label.py
@@ -177,6 +177,7 @@ def test_response_codes(self, kube_apis, ingress_controller, backend_setup, expe
 
 
 @pytest.mark.vs
+@pytest.mark.vs_responses
 @pytest.mark.parametrize(
     "crd_ingress_controller, expected_responses",
     [