Mark vsr external route events as xfail; use single syslog pod for AP…

… sec log tests.

.gitlab-ci.yml

@@ -1,4 +1,4 @@
 include:
   - project: "f5/nginx/kic/kic-pipelines"
     file: "/include/ingress-controller.yml"
-    ref: "try-gke-pipeline-changes"
+    ref: "master"

tests/suite/resources_utils.py

@@ -320,6 +320,19 @@ def get_pods_amount(v1: CoreV1Api, namespace) -> int:
     pods = v1.list_namespaced_pod(namespace)
     return 0 if not pods.items else len(pods.items)
 
+def get_pod_name_that_contains(v1: CoreV1Api, namespace, contains_string) -> str:
+    """
+    Get an amount of pods.
+
+    :param v1: CoreV1Api
+    :param namespace: namespace
+    :param contains_string: string to search on
+    :return: string
+    """
+    for item in v1.list_namespaced_pod(namespace).items:
+        if contains_string in item.metadata.name:
+            return item.metadata.name
+    return ""
 
 def create_service_from_yaml(v1: CoreV1Api, namespace, yaml_manifest) -> str:
     """
@@ -808,6 +821,28 @@ def get_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace) -> str:
     return result_conf
 
 
+def clear_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace):
+    """
+    Execute 'cat /dev/null > file_path' command in a pod.
+
+    :param v1: CoreV1Api
+    :param pod_name: pod name
+    :param pod_namespace: pod namespace
+    :param file_path: an absolute path to a file in the pod
+    """
+    command = ["cat /dev/null > ", file_path]
+    resp = stream(
+        v1.connect_get_namespaced_pod_exec,
+        pod_name,
+        pod_namespace,
+        command=command,
+        stderr=True,
+        stdin=False,
+        stdout=True,
+        tty=False,
+    )
+
+
 def get_ingress_nginx_template_conf(
     v1: CoreV1Api, ingress_namespace, ingress_name, pod_name, pod_namespace
 ) -> str:

tests/suite/test_app_protect_integration.py

@@ -17,10 +17,10 @@
                                    get_file_contents, get_first_pod_name,
                                    get_ingress_nginx_template_conf,
                                    get_last_reload_time, get_pods_amount,
-                                   get_service_endpoint, get_test_file_name,
+                                   clear_file_contents, get_test_file_name,
                                    scale_deployment, wait_before_test,
                                    wait_until_all_pods_are_ready,
-                                   write_to_json)
+                                   write_to_json, get_pod_name_that_contains)
 from suite.yaml_utils import get_first_ingress_host_from_yaml
 
 src_ing_yaml = f"{TEST_DATA}/appprotect/appprotect-ingress.yaml"
@@ -84,8 +84,13 @@ def appprotect_setup(
     src_pol_yaml = f"{TEST_DATA}/appprotect/{ap_policy}.yaml"
     pol_name = create_ap_policy_from_yaml(kube_apis.custom_objects, src_pol_yaml, test_namespace)
 
+    print("------------------------- Deploy syslog server ---------------------------")
+    src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
+    create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
+
     def fin():
         print("Clean up:")
+        delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
         delete_ap_policy(kube_apis.custom_objects, pol_name, test_namespace)
         delete_ap_logconf(kube_apis.custom_objects, log_name, test_namespace)
         delete_common_app(kube_apis, "simple", test_namespace)
@@ -310,15 +315,9 @@ def test_ap_sec_logs_on(
         """
         Test corresponding log entries with correct policy (includes setting up a syslog server as defined in syslog.yaml)
         """
-        src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
         log_loc = "/var/log/messages"
-
-        create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
-
         syslog_dst = f"syslog-svc.{test_namespace}"
-
-        # items[-1] because syslog pod is last one to spin-up
-        syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
+        syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog-")
 
         create_ingress_with_ap_annotations(
             kube_apis, src_ing_yaml, test_namespace, ap_policy, "True", "True", f"{syslog_dst}:514"
@@ -355,7 +354,7 @@ def test_ap_sec_logs_on(
         log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
 
         delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
-        delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
+        clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
 
         assert_invalid_responses(response_block)
         assert (
@@ -385,15 +384,8 @@ def test_ap_pod_startup(
         """
         Log pod startup time while scaling up from 0 to 1
         """
-        src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
-        create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
-
         syslog_dst = f"syslog-svc.{test_namespace}"
 
-        # FIXME this is not used
-        # items[-1] because syslog pod is last one to spin-up
-        # syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
-
         create_ingress_with_ap_annotations(
             kube_apis, src_ing_yaml, test_namespace, ap_policy, "True", "True", f"{syslog_dst}:514"
         )
@@ -409,31 +401,27 @@ def test_ap_pod_startup(
             wait_before_test()
         num = scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 1)
         delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
-        delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
 
         assert num is None
 
-    @pytest.mark.ciara
     @pytest.mark.flaky(max_runs=3)
     def test_ap_multi_sec_logs(
         self, request, kube_apis, crd_ingress_controller_with_ap, appprotect_setup, test_namespace
     ):
         """
         Test corresponding log entries with multiple log destinations (in this case, two syslog servers)
         """
-        src_syslog_yaml = f"{TEST_DATA}/appprotect/syslog.yaml"
         src_syslog2_yaml = f"{TEST_DATA}/appprotect/syslog2.yaml"
         log_loc = "/var/log/messages"
 
-        print("Create two syslog servers")
-        create_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
+        print("Create a second syslog server")
         create_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
 
         syslog_dst = f"syslog-svc.{test_namespace}"
         syslog2_dst = f"syslog2-svc.{test_namespace}"
 
-        syslog_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-2].metadata.name
-        syslog2_pod = kube_apis.v1.list_namespaced_pod(test_namespace).items[-1].metadata.name
+        syslog_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog-")
+        syslog2_pod = get_pod_name_that_contains(kube_apis.v1, test_namespace, "syslog2")
 
         with open(src_ing_yaml) as f:
             doc = yaml.safe_load(f)
@@ -453,44 +441,39 @@ def test_ap_multi_sec_logs(
                 "appprotect.f5.com/app-protect-security-log-destination"
             ] = f"syslog:server={syslog_dst}:514,syslog:server={syslog2_dst}:514"
 
-        try:
-            create_ingress(kube_apis.networking_v1, test_namespace, doc)
+        create_ingress(kube_apis.networking_v1, test_namespace, doc)
 
-            ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
+        ingress_host = get_first_ingress_host_from_yaml(src_ing_yaml)
 
-            ensure_response_from_backend(appprotect_setup.req_url, ingress_host, check404=True)
+        wait_before_test(30)
+        ensure_response_from_backend(appprotect_setup.req_url, ingress_host, check404=True)
 
-            print("----------------------- Send request ----------------------")
-            response = requests.get(
-                appprotect_setup.req_url + "/<script>", headers={"host": ingress_host}, verify=False
-            )
-            print(response.text)
-            log_contents = ""
-            log2_contents = ""
-            retry = 0
-            while (
-                "ASM:attack_type" not in log_contents
-                and "ASM:attack_type" not in log2_contents
-                and retry <= 60
-            ):
-                log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
-                log2_contents = get_file_contents(kube_apis.v1, log_loc, syslog2_pod, test_namespace)
-                retry += 1
-                wait_before_test(1)
-                print(f"Security log not updated, retrying... #{retry}")
-
-            reload_ms = get_last_reload_time(appprotect_setup.metrics_url, "nginx")
-            print(f"last reload duration: {reload_ms} ms")
-            reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
-        except Exception as ex:
-            delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
-            delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
-            delete_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
-            raise ex
+        print("----------------------- Send request ----------------------")
+        response = requests.get(
+            appprotect_setup.req_url + "/<script>", headers={"host": ingress_host}, verify=False
+        )
+        print(response.text)
+        log_contents = ""
+        log2_contents = ""
+        retry = 0
+        while (
+            "ASM:attack_type" not in log_contents
+            and "ASM:attack_type" not in log2_contents
+            and retry <= 60
+        ):
+            log_contents = get_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
+            log2_contents = get_file_contents(kube_apis.v1, log_loc, syslog2_pod, test_namespace)
+            retry += 1
+            wait_before_test(1)
+            print(f"Security log not updated, retrying... #{retry}")
+
+        reload_ms = get_last_reload_time(appprotect_setup.metrics_url, "nginx")
+        print(f"last reload duration: {reload_ms} ms")
+        reload_times[f"{request.node.name}"] = f"last reload duration: {reload_ms} ms"
 
         delete_items_from_yaml(kube_apis, src_ing_yaml, test_namespace)
-        delete_items_from_yaml(kube_apis, src_syslog_yaml, test_namespace)
         delete_items_from_yaml(kube_apis, src_syslog2_yaml, test_namespace)
+        clear_file_contents(kube_apis.v1, log_loc, syslog_pod, test_namespace)
 
         assert_invalid_responses(response)
         # check logs in dest. #1 i.e. syslog server #1

tests/suite/test_v_s_route.py

@@ -40,6 +40,7 @@ def assert_locations_not_in_config(config, paths):
                            {"example": "virtual-server-route"})],
                          indirect=True)
 class TestVirtualServerRoute:
+    @pytest.mark.flaky(max_runs=3)
     def test_responses_and_events_in_flow(self, kube_apis,
                                           ingress_controller_prerequisites,
                                           crd_ingress_controller,

tests/suite/test_v_s_route_externalname.py

@@ -129,7 +129,7 @@ def test_template_config(self, kube_apis,
         assert f"server {vsr_externalname_setup.external_host}:80 max_fails=1 fail_timeout=10s max_conns=0 resolve;"\
                in initial_config
 
-    @pytest.mark.flaky(max_runs=3)
+    @pytest.mark.xfail
     def test_events_flows(self, kube_apis,
                           ingress_controller_prerequisites,
                           crd_ingress_controller,