split appProtect tests to speed up pipeline (#4784)

* split appProtect tests to speed up pipeline

.github/workflows/ci.yml

@@ -264,7 +264,11 @@ jobs:
                                                 {\"image\": \"alpine-plus\", \"marker\":\"ingresses\"}, \
                                                 {\"image\": \"alpine-plus\", \"marker\": \"vsr\"}, \
                                                 {\"image\": \"ubi-plus\", \"marker\": \"policies\"}, \
-                                                {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_integration\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_allow\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_waf_policies and not appprotect_waf_policies_allow'\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_grpc\"}, \
+                                                {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_watch or appprotect_batch'\"}, \
                                                 {\"image\": \"debian-plus-nap\", \"marker\": \"dos\"}], \
                                               \"k8s\": [\"${{ needs.checks.outputs.k8s_latest }}\"]}" >> $GITHUB_OUTPUT
           else

pyproject.toml

@@ -24,6 +24,13 @@ addopts = "--tb=native -ra --disable-warnings -x -l --profile -v"
 log_cli = true
 markers =[
     "appprotect",
+    "appprotect_integration",
+    "appprotect_waf_policies",
+    "appprotect_waf_policies_allow",
+    "appprotect_waf_policies_block",
+    "appprotect_waf_policies_grpc",
+    "appprotect_watch",
+    "appprotect_batch",
     "dos",
     "ingresses",
     "policies",

tests/suite/test_app_protect_integration.py

@@ -147,6 +147,7 @@ def assert_valid_responses(response) -> None:
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_integration
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [

tests/suite/test_app_protect_waf_policies.py

@@ -110,6 +110,7 @@ def assert_valid_responses(response) -> None:
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_waf_policies
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap, virtual_server_setup",
     [
@@ -218,6 +219,7 @@ def test_ap_waf_policy_block(
         else:
             pytest.fail(f"Invalid arguments")
 
+    @pytest.mark.appprotect_waf_policies_allow
     @pytest.mark.parametrize(
         "vs_src, waf",
         [
@@ -366,6 +368,7 @@ def test_ap_waf_policy_multi_logs(
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_waf_policies
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap, v_s_route_setup",
     [
@@ -397,6 +400,7 @@ def restore_default_vsr(self, kube_apis, v_s_route_setup) -> None:
         )
         wait_before_test()
 
+    @pytest.mark.appprotect_waf_policies_block
     @pytest.mark.parametrize(
         "ap_enable",
         [

tests/suite/test_app_protect_waf_policies_grpc.py

@@ -208,6 +208,7 @@ def grpc_waf_allow(kube_apis, test_namespace, public_ip, vs_host, port_ssl):
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_waf_policies_grpc
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [
@@ -302,6 +303,7 @@ def test_responses_grpc_allow(
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_waf_policies_grpc
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [

tests/suite/test_app_protect_watch_namespace.py

@@ -125,6 +125,7 @@ def fin():
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_watch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [{"extra_args": [f"-enable-custom-resources", f"-enable-app-protect", f"-enable-prometheus-metrics"]}],
@@ -157,6 +158,7 @@ def test_responses(self, request, kube_apis, crd_ingress_controller_with_ap, bac
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_watch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [

tests/suite/test_app_protect_watch_namespace_label.py

@@ -124,6 +124,7 @@ def fin():
 
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.appprotect
+@pytest.mark.appprotect_watch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [

tests/suite/test_batch_reloads.py

@@ -216,6 +216,7 @@ def fin():
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.batch_start
 @pytest.mark.appprotect
+@pytest.mark.appprotect_batch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [
@@ -373,6 +374,7 @@ def fin():
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.batch_start
 @pytest.mark.appprotect
+@pytest.mark.appprotect_batch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap, virtual_server_setup",
     [

tests/suite/test_batch_startup_times.py

@@ -216,6 +216,7 @@ def fin():
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.batch_start
 @pytest.mark.appprotect
+@pytest.mark.appprotect_batch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap",
     [
@@ -379,6 +380,7 @@ def fin():
 @pytest.mark.skip_for_nginx_oss
 @pytest.mark.batch_start
 @pytest.mark.appprotect
+@pytest.mark.appprotect_batch
 @pytest.mark.parametrize(
     "crd_ingress_controller_with_ap, virtual_server_setup",
     [