Add namespace support for k8s performance test

Signed-off-by: Cathy Zhang <cathy.zhang@intel.com>

evals/benchmark/benchmark.py

@@ -40,6 +40,10 @@ def extract_test_case_data(content):
     # Extract test suite configuration
     test_suite_config = content.get("test_suite_config", {})
 
+    # Ensure the namespace is a string before calling strip()
+    raw_namespace = test_suite_config.get("namespace")
+    namespace = (raw_namespace.strip() if isinstance(raw_namespace, str) else "") or "default"
+
     return {
         "examples": test_suite_config.get("examples", []),
         "warm_ups": test_suite_config.get("warm_ups", 0),
@@ -55,6 +59,7 @@ def extract_test_case_data(content):
         "load_shape": test_suite_config.get("load_shape"),
         "query_timeout": test_suite_config.get("query_timeout", 120),
         "seed": test_suite_config.get("seed", None),
+        "namespace": namespace,
         "all_case_data": {
             example: content["test_cases"].get(example, {}) for example in test_suite_config.get("examples", [])
         },
@@ -90,7 +95,7 @@ def create_run_yaml_content(service, base_url, bench_target, test_phase, num_que
                 "host": base_url,
                 "stop-timeout": test_params["query_timeout"],
                 "processes": 2,
-                "namespace": "default",
+                "namespace": test_params["namespace"],
                 "bench-target": bench_target,
                 "service-metric-collect": test_params["collect_service_metric"],
                 "service-list": service.get("service_list", []),
@@ -200,7 +205,7 @@ def create_and_save_run_yaml(example, deployment_type, service_type, service, ba
     return run_yaml_paths
 
 
-def get_service_ip(service_name, deployment_type="k8s", service_ip=None, service_port=None):
+def get_service_ip(service_name, deployment_type="k8s", service_ip=None, service_port=None, namespace="default"):
     """Get the service IP and port based on the deployment type.
 
     Args:
@@ -214,7 +219,7 @@ def get_service_ip(service_name, deployment_type="k8s", service_ip=None, service
     """
     if deployment_type == "k8s":
         # Kubernetes IP and port retrieval logic
-        svc_ip, port = get_service_cluster_ip(service_name)
+        svc_ip, port = get_service_cluster_ip(service_name, namespace)
     elif deployment_type == "docker":
         # For Docker deployment, service_ip and service_port must be specified
         if not service_ip or not service_port:
@@ -239,7 +244,11 @@ def run_service_test(example, service_type, service, test_suite_config):
 
     # Get the service IP and port based on deployment type
     svc_ip, port = get_service_ip(
-        service_name, deployment_type, test_suite_config.get("service_ip"), test_suite_config.get("service_port")
+        service_name,
+        deployment_type,
+        test_suite_config.get("service_ip"),
+        test_suite_config.get("service_port"),
+        test_suite_config.get("namespace")
     )
 
     base_url = f"http://{svc_ip}:{port}"
@@ -311,6 +320,7 @@ def run_benchmark(report=False):
         "query_timeout": parsed_data["query_timeout"],
         "warm_ups": parsed_data["warm_ups"],
         "seed": parsed_data["seed"],
+        "namespace": parsed_data["namespace"],
     }
     check_test_suite_config(test_suite_config)
 

evals/benchmark/benchmark.yaml

@@ -23,6 +23,7 @@ test_suite_config: # Overall configuration settings for the test suite
         concurrent_level: 4      # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users
       poisson:                 # Poisson load shape specific parameters, activate only if load_shape is poisson
         arrival-rate: 1.0        # Request arrival rate
+  namespace: "" # Fill the user-defined namespace. Otherwise, it will be default.
 
 test_cases:
   chatqna:

evals/benchmark/utils.py

@@ -29,33 +29,29 @@ def write_json(data, filename):
         logging.error(f"Failed to write {filename}: {e}")
         return False
 
+from kubernetes import client, config
 
-def get_service_cluster_ip(service_name):
-    try:
-        # Run the kubectl command to get the services
-        result = subprocess.run(["kubectl", "get", "svc"], capture_output=True, text=True, check=True)
-
-        # Parse the output
-        lines = result.stdout.splitlines()
-        headers = lines[0].split()
+def get_service_cluster_ip(service_name, namespace="default"):
+    # Load the Kubernetes configuration
+    config.load_kube_config()  # or use config.load_incluster_config() if running inside a Kubernetes pod
 
-        # Find the indices for the columns we are interested in
-        name_idx = headers.index("NAME")
-        cluster_ip_idx = headers.index("CLUSTER-IP")
-        port_idx = headers.index("PORT(S)")
+    # Create an API client for the core API (which handles services)
+    v1 = client.CoreV1Api()
 
-        for line in lines[1:]:
-            columns = line.split()
-            if columns[name_idx] == service_name:
-                cluster_ip = columns[cluster_ip_idx]
-                ports = columns[port_idx]
+    try:
+        # Get the service object
+        service = v1.read_namespaced_service(name=service_name, namespace=namespace)
 
-                main_part = ports.split("/")[0]
-                port = main_part.split(":")[0]
-                return cluster_ip, port
+        # Extract the Cluster IP
+        cluster_ip = service.spec.cluster_ip
 
-        raise ValueError(f"Service {service_name} not found.")
+        # Extract the port number (assuming the first port, modify if necessary)
+        if service.spec.ports:
+            port_number = service.spec.ports[0].port  # Get the first port number
+        else:
+            port_number = None
 
-    except subprocess.CalledProcessError as e:
-        print(f"Error running kubectl command: {e}")
+        return cluster_ip, port_number
+    except client.exceptions.ApiException as e:
+        print(f"Error fetching service: {e}")
         return None