Add startup test for 1 VS multiple VSRs

tests/data/startup/virtual-server-routes/route.yaml

@@ -0,0 +1,11 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServerRoute
+metadata:
+  name: route
+spec:
+  host: example.com
+  subroutes:
+  - path: "/route"
+    action:
+      return:
+        body: "Hello World\n"

tests/data/startup/virtual-server-routes/virtual-server.yaml

@@ -0,0 +1,7 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: virtual-server
+spec:
+  host: example.com
+  routes:

tests/suite/custom_resources_utils.py

@@ -260,15 +260,31 @@ def create_virtual_server_from_yaml(
     print("Create a VirtualServer:")
     with open(yaml_manifest) as f:
         dep = yaml.safe_load(f)
+
+    return create_virtual_server(custom_objects, dep, namespace)
+
+
+def create_virtual_server(
+    custom_objects: CustomObjectsApi, vs, namespace
+) -> str:
+    """
+    Create a VirtualServer.
+
+    :param custom_objects: CustomObjectsApi
+    :param vs: a VirtualServer
+    :param namespace:
+    :return: str
+    """
+    print("Create a VirtualServer:")
     try:
         custom_objects.create_namespaced_custom_object(
-            "k8s.nginx.org", "v1", namespace, "virtualservers", dep
+            "k8s.nginx.org", "v1", namespace, "virtualservers", vs
         )
-        print(f"VirtualServer created with name '{dep['metadata']['name']}'")
-        return dep["metadata"]["name"]
+        print(f"VirtualServer created with name '{vs['metadata']['name']}'")
+        return vs["metadata"]["name"]
     except ApiException as ex:
         logging.exception(
-            f"Exception: {ex} occurred while creating VirtualServer: {dep['metadata']['name']}"
+            f"Exception: {ex} occurred while creating VirtualServer: {vs['metadata']['name']}"
         )
         raise
 
@@ -697,11 +713,23 @@ def create_v_s_route_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest,
     with open(yaml_manifest) as f:
         dep = yaml.safe_load(f)
 
+    return create_v_s_route(custom_objects, dep, namespace)
+
+def create_v_s_route(custom_objects: CustomObjectsApi, vsr, namespace) -> str:
+    """
+    Create a VirtualServerRoute.
+
+    :param custom_objects: CustomObjectsApi
+    :param vsr: a VirtualServerRoute
+    :param namespace:
+    :return: str
+    """
+    print("Create a VirtualServerRoute:")
     custom_objects.create_namespaced_custom_object(
-        "k8s.nginx.org", "v1", namespace, "virtualserverroutes", dep
+        "k8s.nginx.org", "v1", namespace, "virtualserverroutes", vsr
     )
-    print(f"VirtualServerRoute created with a name '{dep['metadata']['name']}'")
-    return dep["metadata"]["name"]
+    print(f"VirtualServerRoute created with a name '{vsr['metadata']['name']}'")
+    return vsr["metadata"]["name"]
 
 
 def patch_v_s_route(custom_objects: CustomObjectsApi, name, namespace, body) -> str:

tests/suite/resources_utils.py

@@ -1337,6 +1337,15 @@ def get_total_vs(req_url, ingress_class) -> str:
     return parse_metric_data(resp_content, metric_string)
 
 
+def get_total_vsr(req_url, ingress_class) -> str:
+    # return total number of virtualserverroutes in specified ingress class
+    ensure_connection(req_url, 200)
+    resp = requests.get(req_url)
+    resp_content = resp.content.decode("utf-8")
+    metric_string = 'virtualserverroute_resources_total{class="%s"}' % ingress_class
+    return parse_metric_data(resp_content, metric_string)
+
+
 def get_last_reload_status(req_url, ingress_class) -> str:
     # returnb last reload status 0/1
     ensure_connection(req_url, 200)

tests/suite/test_batch_startup_times.py

@@ -1,3 +1,5 @@
+import time
+
 import requests
 import pytest
 import yaml
@@ -26,7 +28,7 @@
     get_total_ingresses,
     get_total_vs,
     get_last_reload_status,
-    get_pods_amount,
+    get_pods_amount, get_total_vsr,
 )
 from suite.custom_resources_utils import (
     create_virtual_server_from_yaml,
@@ -37,6 +39,8 @@
     create_policy_from_yaml,
     create_ap_waf_policy_from_yaml,
     delete_policy,
+    create_virtual_server,
+    create_v_s_route,
 )
 from suite.yaml_utils import get_first_ingress_host_from_yaml
 from settings import TEST_DATA
@@ -503,3 +507,92 @@ def test_ap_waf_policy_vs_batch_start(
         delete_policy(kube_apis.custom_objects, "waf-policy", test_namespace)
 
         assert num is None
+
+
+##############################################################################################################
+
+@pytest.fixture(scope="class")
+def vs_vsr_setup(
+    request,
+    kube_apis,
+    test_namespace,
+    ingress_controller_endpoint,
+):
+    """
+    Deploy one VS with multiple VSRs.
+
+    :param kube_apis: client apis
+    :param test_namespace:
+    :param ingress_controller_endpoint: public endpoint
+    :return:
+    """
+    total_vsr = int(request.config.getoption("--batch-resources"))
+
+    vsr_source = f"{TEST_DATA}/startup/virtual-server-routes/route.yaml"
+
+    with open(vsr_source) as f:
+        vsr = yaml.safe_load(f)
+
+        for i in range(1, total_vsr + 1):
+            vsr["metadata"]["name"] = f"route-{i}"
+            vsr["spec"]["subroutes"][0]["path"] = f"/route-{i}"
+
+            create_v_s_route(kube_apis.custom_objects, vsr, test_namespace)
+
+    vs_source = f"{TEST_DATA}/startup/virtual-server-routes/virtual-server.yaml"
+
+    with open(vs_source) as f:
+        vs = yaml.safe_load(f)
+
+        routes = []
+        for i in range(1, total_vsr + 1):
+            route = {"path": f"/route-{i}", "route": f"route-{i}"}
+            routes.append(route)
+
+        vs["spec"]["routes"] = routes
+        create_virtual_server(kube_apis.custom_objects, vs, test_namespace)
+
+
+@pytest.mark.batch_start
+@pytest.mark.parametrize(
+    "crd_ingress_controller",
+    [
+        pytest.param(
+            {
+                "type": "complete",
+                "extra_args": ["-enable-custom-resources","-enable-prometheus-metrics", "-enable-leader-election=false"]
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestSingleVSMultipleVSRs:
+    def test_startup_time(
+        self,
+        request,
+        kube_apis,
+        ingress_controller_prerequisites,
+        crd_ingress_controller,
+        ingress_controller_endpoint,
+         vs_vsr_setup):
+        """
+        Pod startup time with 1 VS and multiple VSRs.
+        """
+        total_vsr = int(request.config.getoption("--batch-resources"))
+
+        ic_ns = ingress_controller_prerequisites.namespace
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ic_ns, 0)
+        while get_pods_amount(kube_apis.v1, ic_ns) is not 0:
+            print(f"Number of replicas not 0, retrying...")
+            wait_before_test()
+        num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ic_ns, 1)
+
+        metrics_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics"
+
+        assert (
+            get_total_vs(metrics_url, "nginx") == "1"
+            and get_total_vsr(metrics_url, "nginx") == str(total_vsr)
+            and get_last_reload_status(metrics_url, "nginx") == "1"
+        )
+
+        assert num is None