From 1702ec0829a3ab39b948328360877ba99ef37030 Mon Sep 17 00:00:00 2001 From: NucciTheBoss Date: Thu, 24 Aug 2023 14:54:07 -0400 Subject: [PATCH 1/2] tests: Add support for using local charms in integration tests * Removed old references to ETCD. * Deleted helpers module because it was no longer necessary. * Add `--use-local` option to direct integration tests to use local slurmctld operator. Signed-off-by: Jason C. Nucciarone --- tests/integration/conftest.py | 55 ++++++++++++++++++++++++--------- tests/integration/helpers.py | 34 -------------------- tests/integration/test_charm.py | 28 +++++++---------- tox.ini | 3 +- 4 files changed, 55 insertions(+), 65 deletions(-) delete mode 100644 tests/integration/helpers.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bc80d00..b647f07 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,35 +13,62 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Configure integration test run.""" +"""Configure slurmdbd operator integration tests.""" -import pathlib +import logging +import os +from pathlib import Path +from typing import Union import pytest -from _pytest.config.argparsing import Parser -from helpers import ETCD from pytest_operator.plugin import OpsTest +logger = logging.getLogger(__name__) +SLURMCTLD_DIR = Path(os.getenv("SLURMCTLD_DIR", "../slurmctld-operator")) -def pytest_addoption(parser: Parser) -> None: + +def pytest_addoption(parser) -> None: + parser.addoption( + "--charm-base", + action="store", + default="ubuntu@22.04", + help="Charm base version to use for integration tests", + ) parser.addoption( - "--charm-base", action="store", default="ubuntu@22.04", help="Charm base to test." + "--use-local", + action="store_true", + default=False, + help="Use SLURM operators located on localhost rather than pull from Charmhub", ) @pytest.fixture(scope="module") def charm_base(request) -> str: """Get slurmdbd charm base to use.""" - return request.config.getoption("--charm-base") + return request.config.option.charm_base + + +@pytest.fixture(scope="module") +async def slurmdbd_charm(ops_test: OpsTest) -> Path: + """Pack slurmdbd charm to use for integration tests.""" + return await ops_test.build_charm(".") @pytest.fixture(scope="module") -async def slurmdbd_charm(ops_test: OpsTest): - """Build slurmdbd charm to use for integration tests.""" - charm = await ops_test.build_charm(".") - return charm +async def slurmctld_charm(request, ops_test: OpsTest) -> Union[str, Path]: + """Pack slurmctld charm to use for integration tests when --use-local is specified. + Returns: + `str` "slurmctld" if --use-local not specified or if SLURMD_DIR does not exist. + """ + if request.config.option.use_local: + logger.info("Using local slurmctld operator rather than pulling from Charmhub") + if SLURMCTLD_DIR.exists(): + return await ops_test.build_charm(SLURMCTLD_DIR) + else: + logger.warning( + f"{SLURMCTLD_DIR} not found. " + f"Defaulting to latest/edge slurmctld operator from Charmhub" + ) -def pytest_sessionfinish(session, exitstatus) -> None: - """Clean up repository after test session has completed.""" - pathlib.Path(ETCD).unlink(missing_ok=True) + return "slurmctld" diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py deleted file mode 100644 index e9f2e3a..0000000 --- a/tests/integration/helpers.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers for the slurmdbd integration tests.""" - -import logging -import pathlib -from typing import Dict -from urllib import request - -logger = logging.getLogger(__name__) - -ETCD = "etcd-v3.5.0-linux-amd64.tar.gz" -ETCD_URL = f"https://github.com/etcd-io/etcd/releases/download/v3.5.0/{ETCD}" - - -def get_slurmctld_res() -> Dict[str, pathlib.Path]: - """Get slurmctld resources needed for charm deployment.""" - if not (etcd := pathlib.Path(ETCD)).exists(): - logger.info(f"Getting resource {ETCD} from {ETCD_URL}") - request.urlretrieve(ETCD_URL, etcd) - - return {"etcd": etcd} diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 4929c9b..56e7519 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -13,16 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test slurmdbd charm against other SLURM charms in the latest/edge channel.""" +"""Test slurmdbd charm against other SLURM operators.""" import asyncio import logging -import pathlib -from typing import Any, Coroutine import pytest import tenacity -from helpers import get_slurmctld_res from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) @@ -31,30 +28,31 @@ SLURMCTLD = "slurmctld" DATABASE = "mysql" ROUTER = "mysql-router" +UNIT_NAME = f"{SLURMDBD}/0" @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed @pytest.mark.order(1) async def test_build_and_deploy_against_edge( - ops_test: OpsTest, slurmdbd_charm: Coroutine[Any, Any, pathlib.Path], charm_base: str + ops_test: OpsTest, charm_base: str, slurmdbd_charm, slurmctld_charm ) -> None: """Test that the slurmdbd charm can stabilize against slurmctld and MySQL.""" logger.info(f"Deploying {SLURMDBD} against {SLURMCTLD} and {DATABASE}") - slurmctld_res = get_slurmctld_res() + # Pack charms + slurmdbd, slurmctld = await asyncio.gather(slurmdbd_charm, slurmctld_charm) await asyncio.gather( ops_test.model.deploy( - str(await slurmdbd_charm), + str(slurmdbd), application_name=SLURMDBD, num_units=1, base=charm_base, ), ops_test.model.deploy( - SLURMCTLD, + str(slurmctld), application_name=SLURMCTLD, - channel="edge", + channel="edge" if isinstance(slurmctld, str) else None, num_units=1, - resources=slurmctld_res, base=charm_base, ), ops_test.model.deploy( @@ -72,8 +70,6 @@ async def test_build_and_deploy_against_edge( base="ubuntu@22.04", ), ) - # Attach resources to charms. - await ops_test.juju("attach-resource", SLURMCTLD, f"etcd={slurmctld_res['etcd']}") # Set relations for charmed applications. await ops_test.model.integrate(f"{SLURMDBD}:{SLURMDBD}", f"{SLURMCTLD}:{SLURMDBD}") await ops_test.model.integrate(f"{SLURMDBD}-{ROUTER}:backend-database", f"{DATABASE}:database") @@ -81,7 +77,7 @@ async def test_build_and_deploy_against_edge( # Reduce the update status frequency to accelerate the triggering of deferred events. async with ops_test.fast_forward(): await ops_test.model.wait_for_idle(apps=[SLURMDBD], status="active", timeout=1000) - assert ops_test.model.applications[SLURMDBD].units[0].workload_status == "active" + assert ops_test.model.units.get(UNIT_NAME).workload_status == "active" @pytest.mark.abort_on_fail @@ -94,7 +90,7 @@ async def test_build_and_deploy_against_edge( async def test_slurmdbd_is_active(ops_test: OpsTest) -> None: """Test that slurmdbd is active inside Juju unit.""" logger.info("Checking that slurmdbd daemon is active inside unit") - slurmdbd_unit = ops_test.model.applications[SLURMDBD].units[0] + slurmdbd_unit = ops_test.model.units.get(UNIT_NAME) res = (await slurmdbd_unit.ssh("systemctl is-active slurmdbd")).strip("\n") assert res == "active" @@ -109,7 +105,7 @@ async def test_slurmdbd_is_active(ops_test: OpsTest) -> None: async def test_slurmdbd_port_listen(ops_test: OpsTest) -> None: """Test that slurmdbd is listening on port 6819.""" logger.info("Checking that slurmdbd is listening on port 6819") - slurmdbd_unit = ops_test.model.applications[SLURMDBD].units[0] + slurmdbd_unit = ops_test.model.units.get(UNIT_NAME) res = await slurmdbd_unit.ssh("sudo lsof -t -n -iTCP:6819 -sTCP:LISTEN") assert res != "" @@ -124,6 +120,6 @@ async def test_slurmdbd_port_listen(ops_test: OpsTest) -> None: async def test_munge_is_active(ops_test: OpsTest) -> None: """Test that munge is active inside Juju unit.""" logger.info("Checking that munge is active inside Juju unit") - slurmdbd_unit = ops_test.model.applications[SLURMDBD].units[0] + slurmdbd_unit = ops_test.model.units.get(UNIT_NAME) res = (await slurmdbd_unit.ssh("systemctl is-active munge")).strip("\n") assert res == "active" diff --git a/tox.ini b/tox.ini index 84324de..eaeb37f 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ passenv = PYTHONPATH CHARM_BUILD_DIR MODEL_SETTINGS + SLURMCTLD_DIR [testenv:fmt] description = Apply coding style standards to code @@ -56,7 +57,7 @@ commands = [testenv:integration] description = Run integration tests deps = - juju==3.1.0.1 + juju pytest==7.2.0 pytest-operator==0.26.0 pytest-order==1.1.0 From 2fe34c680bac5e65c95730dc2447af52877aa675 Mon Sep 17 00:00:00 2001 From: NucciTheBoss Date: Thu, 24 Aug 2023 15:33:16 -0400 Subject: [PATCH 2/2] chore(lint): Fix confeditor so it passes lint * Needed to replace type(...) == type calls with isinstance(...) Using type(...) == type instead of isinstance violates lint rule E721. Signed-off-by: Jason C. Nucciarone --- src/utils/confeditor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/confeditor.py b/src/utils/confeditor.py index cc74a54..4bc7ce4 100644 --- a/src/utils/confeditor.py +++ b/src/utils/confeditor.py @@ -525,7 +525,7 @@ def auth_alt_types(self) -> Optional[List[str]]: @auth_alt_types.setter def auth_alt_types(self, value: Union[str, List[str]]) -> None: """Set configuration value for parameter `AuthAltTypes`.""" - value = [value] if type(value) == str else value + value = [value] if isinstance(value, str) else value self._metadata[_SlurmdbdToken.AuthAltTypes] = ",".join(value) @auth_alt_types.deleter @@ -600,7 +600,7 @@ def communication_parameters(self) -> Optional[List[str]]: @communication_parameters.setter def communication_parameters(self, value: Union[str, List[str]]) -> None: """Set configuration value for parameter `CommunicationParameters`.""" - value = [value] if type(value) == str else value + value = [value] if isinstance(value, str) else value self._metadata[_SlurmdbdToken.CommunicationParameters] = ",".join(value) @communication_parameters.deleter @@ -685,7 +685,7 @@ def debug_flags(self) -> Optional[List[str]]: @debug_flags.setter def debug_flags(self, value: Union[str, List[str]]) -> None: """Set configuration value for parameter `DebugFlags`.""" - value = [value] if type(value) == str else value + value = [value] if isinstance(value, str) else value for flag in value: _check_debug_flag(flag) self._metadata[_SlurmdbdToken.DebugFlags] = ",".join(value) @@ -820,7 +820,7 @@ def parameters(self) -> Optional[List[str]]: @parameters.setter def parameters(self, value: Union[str, List[str]]) -> None: """Set configuration value for parameter `Parameters`.""" - value = [value] if type(value) == str else value + value = [value] if isinstance(value, str) else value self._metadata[_SlurmdbdToken.Parameters] = ",".join(value) @parameters.deleter @@ -855,7 +855,7 @@ def plugin_dir(self) -> Optional[List[str]]: @plugin_dir.setter def plugin_dir(self, value: Union[str, List[str]]) -> None: """Set configuration value for parameter `PluginDir`.""" - value = [value] if type(value) == str else value + value = [value] if isinstance(value, str) else value self._metadata[_SlurmdbdToken.PluginDir] = ":".join(value) @plugin_dir.deleter @@ -875,7 +875,7 @@ def private_data(self) -> Optional[List[str]]: @private_data.setter def private_data(self, value: Union[str, List[str]]) -> None: """Set configuration value for parameter `PrivateData`.""" - value = [value] if type(value) == str else value + value = [value] if isinstance(value, str) else value for data in value: _check_private_data(data) self._metadata[_SlurmdbdToken.PrivateData] = ",".join(value)