From a83e45f0f5703c824ac0cc424a41b41bada5e98a Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 29 Jul 2024 12:38:59 +0300 Subject: [PATCH 1/5] Formatting fixes to util.platform_util. Signed-off-by: Nashwan Azhari --- k8s_test_harness/util/platform_util.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/k8s_test_harness/util/platform_util.py b/k8s_test_harness/util/platform_util.py index 0f25a6a..ce32626 100644 --- a/k8s_test_harness/util/platform_util.py +++ b/k8s_test_harness/util/platform_util.py @@ -8,7 +8,6 @@ import platform - ROCKCRAFT_PLATFORM_AMD64 = "amd64" ROCKCRAFT_PLATFORM_I386 = "i386" ROCKCRAFT_PLATFORM_ARM64 = "arm64" @@ -17,12 +16,12 @@ _PYTHON_MACHINE_TO_ROCKCRAFT_PLATFORM_ARCHITECTURE_MAP = { "x86": ROCKCRAFT_PLATFORM_I386, "x86_64": ROCKCRAFT_PLATFORM_AMD64, - "arm64": ROCKCRAFT_PLATFORM_ARM64 + "arm64": ROCKCRAFT_PLATFORM_ARM64, } def get_current_rockcraft_platform_architecture() -> str: - """ Returns a string containing the rockcraft-specific platform + """Returns a string containing the rockcraft-specific platform architecture label of the currently running process. https://documentation.ubuntu.com/rockcraft/en/latest/reference/rockcraft.yaml/#platforms @@ -33,13 +32,12 @@ def get_current_rockcraft_platform_architecture() -> str: machine = platform.machine() if not machine: - raise OSError( - "Failed to get current platform through `platform.machine()`.") + raise OSError("Failed to get current platform through `platform.machine()`.") if machine not in _PYTHON_MACHINE_TO_ROCKCRAFT_PLATFORM_ARCHITECTURE_MAP: raise ValueError( f"Unknown platform machine type '{machine}'. Known values are: " - f"{list(_PYTHON_MACHINE_TO_ROCKCRAFT_PLATFORM_ARCHITECTURE_MAP)}") + f"{list(_PYTHON_MACHINE_TO_ROCKCRAFT_PLATFORM_ARCHITECTURE_MAP)}" + ) return _PYTHON_MACHINE_TO_ROCKCRAFT_PLATFORM_ARCHITECTURE_MAP[machine] - From c459537f5a13d2b16601a218b3b0c19156e707a9 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 29 Jul 2024 12:42:09 +0300 Subject: [PATCH 2/5] Add retry-related kwargs to k8s_util.wait_for_* functions. Signed-off-by: Nashwan Azhari --- k8s_test_harness/util/k8s_util.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/k8s_test_harness/util/k8s_util.py b/k8s_test_harness/util/k8s_util.py index 8cc1832..4381582 100644 --- a/k8s_test_harness/util/k8s_util.py +++ b/k8s_test_harness/util/k8s_util.py @@ -129,9 +129,11 @@ def wait_for_resource( name: str, namespace: str = constants.K8S_NS_DEFAULT, condition: str = constants.K8S_CONDITION_AVAILABLE, + retry_times: int = 5, + retry_delay_s: int = 10, ): """Waits for the given resource to reach the given condition.""" - exec_util.stubbornly(retries=5, delay_s=1).on(instance).exec( + exec_util.stubbornly(retries=retry_times, delay_s=retry_delay_s).on(instance).exec( [ "k8s", "kubectl", @@ -142,7 +144,7 @@ def wait_for_resource( resource_type, name, "--timeout", - "60s", + "1s", ] ) @@ -152,16 +154,30 @@ def wait_for_deployment( name: str, namespace: str = constants.K8S_NS_DEFAULT, condition: str = constants.K8S_CONDITION_AVAILABLE, + retry_times: int = 5, + retry_delay_s: int = 10, ): """Waits for the given deployment to reach the given condition.""" - wait_for_resource(instance, constants.K8S_DEPLOYMENT, name, namespace, condition) + wait_for_resource( + instance, + constants.K8S_DEPLOYMENT, + name, + namespace=namespace, + condition=condition, + retry_times=retry_times, + retry_delay_s=retry_delay_s, + ) def wait_for_daemonset( - instance: harness.Instance, name: str, namespace: str = constants.K8S_NS_DEFAULT + instance: harness.Instance, + name: str, + namespace: str = constants.K8S_NS_DEFAULT, + retry_times: int = 5, + retry_delay_s: int = 10, ): """Waits for the given daemonset to become available.""" - exec_util.stubbornly(retries=5, delay_s=1).on(instance).exec( + exec_util.stubbornly(retries=retry_times, delay_s=retry_delay_s).on(instance).exec( [ "k8s", "kubectl", @@ -172,7 +188,7 @@ def wait_for_daemonset( constants.K8S_DAEMONSET, name, "--timeout", - "60s", + "1s", ] ) From a215b178a7c497d5bf9c803909593f64ae2c4cd2 Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 29 Jul 2024 12:42:33 +0300 Subject: [PATCH 3/5] Add util.k8s_utils.wait_for_stateful_set(). Signed-off-by: Nashwan Azhari --- k8s_test_harness/util/constants.py | 1 + k8s_test_harness/util/k8s_util.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/k8s_test_harness/util/constants.py b/k8s_test_harness/util/constants.py index 515dd38..225046c 100644 --- a/k8s_test_harness/util/constants.py +++ b/k8s_test_harness/util/constants.py @@ -8,6 +8,7 @@ K8S_DAEMONSET = "daemonset.apps" K8S_DEPLOYMENT = "deployment.apps" +K8S_STATEFULSET = "statefulset.apps" K8S_CONDITION_AVAILABLE = "Available" K8S_CONDITION_READY = "Ready" diff --git a/k8s_test_harness/util/k8s_util.py b/k8s_test_harness/util/k8s_util.py index 4381582..01277a4 100644 --- a/k8s_test_harness/util/k8s_util.py +++ b/k8s_test_harness/util/k8s_util.py @@ -193,6 +193,30 @@ def wait_for_daemonset( ) +def wait_for_statefulset( + instance: harness.Instance, + name: str, + namespace: str = constants.K8S_NS_DEFAULT, + retry_times: int = 5, + retry_delay_s: int = 10, +): + """Waits for the given daemonset to become available.""" + exec_util.stubbornly(retries=retry_times, delay_s=retry_delay_s).on(instance).exec( + [ + "k8s", + "kubectl", + "rollout", + "status", + "--namespace", + namespace, + constants.K8S_STATEFULSET, + name, + "--timeout", + "1s", + ] + ) + + def get_helm_install_command( name: str, chart_name: str, From a2f6e7b3f0bafebe81ace1349aa1806a06cac2ae Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 29 Jul 2024 18:08:47 +0300 Subject: [PATCH 4/5] Fix default delays in k8s_util functions. Signed-off-by: Nashwan Azhari --- k8s_test_harness/util/k8s_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s_test_harness/util/k8s_util.py b/k8s_test_harness/util/k8s_util.py index 01277a4..1b6aa08 100644 --- a/k8s_test_harness/util/k8s_util.py +++ b/k8s_test_harness/util/k8s_util.py @@ -130,7 +130,7 @@ def wait_for_resource( namespace: str = constants.K8S_NS_DEFAULT, condition: str = constants.K8S_CONDITION_AVAILABLE, retry_times: int = 5, - retry_delay_s: int = 10, + retry_delay_s: int = 60, ): """Waits for the given resource to reach the given condition.""" exec_util.stubbornly(retries=retry_times, delay_s=retry_delay_s).on(instance).exec( @@ -155,7 +155,7 @@ def wait_for_deployment( namespace: str = constants.K8S_NS_DEFAULT, condition: str = constants.K8S_CONDITION_AVAILABLE, retry_times: int = 5, - retry_delay_s: int = 10, + retry_delay_s: int = 60, ): """Waits for the given deployment to reach the given condition.""" wait_for_resource( @@ -174,7 +174,7 @@ def wait_for_daemonset( name: str, namespace: str = constants.K8S_NS_DEFAULT, retry_times: int = 5, - retry_delay_s: int = 10, + retry_delay_s: int = 60, ): """Waits for the given daemonset to become available.""" exec_util.stubbornly(retries=retry_times, delay_s=retry_delay_s).on(instance).exec( @@ -198,7 +198,7 @@ def wait_for_statefulset( name: str, namespace: str = constants.K8S_NS_DEFAULT, retry_times: int = 5, - retry_delay_s: int = 10, + retry_delay_s: int = 60, ): """Waits for the given daemonset to become available.""" exec_util.stubbornly(retries=retry_times, delay_s=retry_delay_s).on(instance).exec( From bb73a1b65172942e01703e5580e2035ed9f5499f Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Mon, 29 Jul 2024 21:38:05 +0300 Subject: [PATCH 5/5] Describe relevant resources on wait_for_*() errors. Signed-off-by: Nashwan Azhari --- k8s_test_harness/util/k8s_util.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/k8s_test_harness/util/k8s_util.py b/k8s_test_harness/util/k8s_util.py index 1b6aa08..072140c 100644 --- a/k8s_test_harness/util/k8s_util.py +++ b/k8s_test_harness/util/k8s_util.py @@ -3,6 +3,7 @@ # See LICENSE file for licensing details # +import functools import itertools import json import logging @@ -32,6 +33,27 @@ def purge_k8s_snap(instance: harness.Instance): instance.exec(["sudo", "snap", "remove", "k8s", "--purge"]) +def describe_resources_on_error(resource_type: str): + def _decorator(fun): + @functools.wraps(fun) + def _inner(instance: harness.Instance, *args, **kwargs): + try: + return fun(instance, *args, **kwargs) + except Exception: + proc = instance.exec( + ["k8s", "kubectl", "describe", resource_type], capture_output=True + ) + LOG.info( + f"### All current '{resource_type}' definitions: " + f"{proc.stdout.decode()}" + ) + raise + + return _inner + + return _decorator + + # Validates that the K8s node is in Ready state. def wait_until_k8s_ready( control_node: harness.Instance, instances: List[harness.Instance] @@ -149,6 +171,8 @@ def wait_for_resource( ) +@describe_resources_on_error("pods") +@describe_resources_on_error("deployment") def wait_for_deployment( instance: harness.Instance, name: str, @@ -169,6 +193,8 @@ def wait_for_deployment( ) +@describe_resources_on_error("pods") +@describe_resources_on_error("daemonsets") def wait_for_daemonset( instance: harness.Instance, name: str, @@ -193,6 +219,8 @@ def wait_for_daemonset( ) +@describe_resources_on_error("pods") +@describe_resources_on_error("statefulsets") def wait_for_statefulset( instance: harness.Instance, name: str,