From 0242fd0d3b5a9855493b110c9223baa9cc8d7c2b Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Thu, 2 Mar 2023 14:35:08 +0100 Subject: [PATCH 1/2] push snap command --- jhack/helpers.py | 60 +++++++++-- jhack/main.py | 9 ++ jhack/snap/push_snap.py | 182 +++++++++++++++++++++++++++++++++ jhack/tests/utils/test_tail.py | 3 +- jhack/utils/tail_charms.py | 40 +------- 5 files changed, 247 insertions(+), 47 deletions(-) create mode 100644 jhack/snap/push_snap.py diff --git a/jhack/helpers.py b/jhack/helpers.py index dc861cd..3d6a8f5 100644 --- a/jhack/helpers.py +++ b/jhack/helpers.py @@ -4,10 +4,12 @@ import os import subprocess import tempfile +from dataclasses import dataclass from functools import lru_cache +from itertools import chain from pathlib import Path from subprocess import PIPE, CalledProcessError, check_call, check_output -from typing import Iterable, List, Literal, Optional, Tuple +from typing import Iterable, List, Literal, Optional, Sequence, Tuple import typer @@ -74,9 +76,9 @@ def _JPopen(args: Tuple[str], wait: bool, **kwargs): # noqa # Env-passing-down Popen proc = subprocess.Popen( args, - env=kwargs.pop("env", os.environ), - stderr=kwargs.pop("stderr", PIPE), - stdout=kwargs.pop("stdout", PIPE), + env=kwargs.get("env", os.environ), + stderr=kwargs.get("stderr", PIPE), + stdout=kwargs.get("stdout", PIPE), **kwargs, ) if wait: @@ -84,7 +86,7 @@ def _JPopen(args: Tuple[str], wait: bool, **kwargs): # noqa # this will presumably only ever branch if wait==True if proc.returncode not in {0, None}: - msg = f"failed to invoke juju command ({args}, {kwargs})" + msg = f"failed to invoke juju command ({' '.join(args)!r}, {kwargs}); exited with {proc.returncode}" if IS_SNAPPED and "ssh client keys" in proc.stderr.read().decode("utf-8"): msg += ( " If you see an ERROR above saying something like " @@ -237,5 +239,49 @@ def fetch_file( local_path.write_bytes(raw) -if __name__ == "__main__": - get_substrate() +@dataclass +class Target: + app: str + unit: int + leader: bool = False + + @staticmethod + def from_name(name: str): + if "/" not in name: + logger.warning( + "invalid target name: expected `/`; " + f"got {name!r}." + ) + app, unit_ = name.split("/") + leader = unit_.endswith("*") + unit = unit_.strip("*") + return Target(app, unit, leader=leader) + + @property + def unit_name(self): + return f"{self.app}/{self.unit}" + + def __hash__(self): + return hash((self.app, self.unit, self.leader)) + + +def get_all_units( + model: str = None, filter_apps: Iterable[str] = None +) -> Sequence[Target]: + def _filter(app): + if filter_apps: + return app in filter_apps + return True + + status = juju_status(json=True, model=model) + # sub charms don't have units or applications + units = list( + chain( + *( + app.get("units", ()) + for app_name, app in status.get("applications", {}).items() + if _filter(app_name) + ) + ) + ) + return tuple(map(Target.from_name, units)) diff --git a/jhack/main.py b/jhack/main.py index d9a99ee..10fde92 100755 --- a/jhack/main.py +++ b/jhack/main.py @@ -34,6 +34,7 @@ def main(): from jhack.jinx.install import install as jinx_install from jhack.jinx.pack import pack as jinx_pack from jhack.logger import LOGLEVEL, logger + from jhack.snap.push_snap import push_snap from jhack.utils import integrate from jhack.utils.event_recorder.client import ( dump_db, @@ -123,12 +124,20 @@ def main(): conf.command(name="default")(print_defaults) conf.command(name="current")(print_current_config) + snap = typer.Typer( + name="snap", + help="""Snap utilities.""", + no_args_is_help=True, + ) + snap.command(name="push")(push_snap) + app.add_typer(conf, no_args_is_help=True) app.add_typer(jinx, no_args_is_help=True) app.add_typer(charm, no_args_is_help=True) app.add_typer(utils, no_args_is_help=True) app.add_typer(replay, no_args_is_help=True) app.add_typer(integration_matrix, no_args_is_help=True) + app.add_typer(snap, no_args_is_help=True) @app.callback() def set_verbose(log: str = None, path: Path = None): diff --git a/jhack/snap/push_snap.py b/jhack/snap/push_snap.py new file mode 100644 index 0000000..ab11321 --- /dev/null +++ b/jhack/snap/push_snap.py @@ -0,0 +1,182 @@ +import os +import sys +from contextlib import contextmanager +from enum import Enum +from pathlib import Path +from subprocess import CalledProcessError, run +from typing import Literal + +import typer + +from jhack.helpers import get_all_units, get_substrate, JPopen +from jhack.logger import logger + +logger = logger.getChild(__file__) + + +@contextmanager +def cwd(dir: Path): + previous_cwd = os.getcwd() + os.chdir(str(dir)) + yield + os.chdir(previous_cwd) + + +class SnapCtlError(RuntimeError): + pass + + +def _push_to_unit(unit_name: str, snap_name: str): + logger.info(f"shelling snap over to {unit_name}...") + cmd = f"juju scp ./{snap_name} {unit_name}:~/" + logger.debug("running: cmd") + + def fail(): + fail_msg = f"Failed scp'ing {snap_name} to {unit_name}" + logger.error(fail_msg, exc_info=True) + return SnapCtlError(fail_msg) + + try: + proc = JPopen(cmd.split(), wait=True) + + except (FileNotFoundError, CalledProcessError) as e: + raise fail() from e + + proc.wait() + out = proc.stdout.read().decode("utf-8") + + if proc.returncode != 0: + err = proc.stderr.read().decode("utf-8") + logger.error(f'stderr = {err}') + raise fail() + + logger.debug(out) + logger.info("\tshelled.") + + +def _clean(unit_name, snap_name, mode): + _mode = " --purge" if mode == "force" else "" + logger.info(f"cleaning up{_mode} {snap_name} in {unit_name}") + cmd = f"juju ssh {unit_name} -- sudo snap remove {snap_name}{_mode}" + # check=False as we don't care if this fails because e.g. the snap was not installed. + out = JPopen(cmd.split(), wait=True).stdout.read().decode("utf-8") + logger.debug(out) + logger.info(f"\tcleaned.") + + +def _install_in_unit( + unit_name: str, snap_name: str, clean: Literal[False, True, "force"] +): + if clean: + _clean(unit_name, snap_name, mode=clean) + + logger.info(f"installing snap in {unit_name}...") + cmd = f"juju ssh {unit_name} -- sudo snap install --dangerous '~/{snap_name}'" + logger.debug(f"command = {cmd}") + + try: + out = JPopen( + cmd.split(), wait=True + ).stdout.read().decode("utf-8") + except CalledProcessError as e: + msg = f"Failed installing {snap_name} in {unit_name}" + logger.error(msg, exc_info=True) + raise SnapCtlError(msg) from e + + logger.debug(out) + logger.info("\tinstalled.") + + +class CleanOpt(str, Enum): + no = False + yes = True + force = "force" + + +def _push_snap( + target: str, + snap: Path = Path("./"), + rebuild: bool = False, + model: str = None, + clean: CleanOpt = CleanOpt.yes, +): + if get_substrate(model) == "k8s": + exit(f"{model or 'this model'} is not a machine model.") + + logger.info(f"snap root={snap}") + with cwd(snap): + if rebuild: + logger.info("snapping...") + try: + run(["snapcraft", "--use-lxd"], capture_output=True, check=True) + except CalledProcessError: + logger.error('failed packing snap.', exc_info=True) + exit("failed packing snap.") + + try: + snap_name = next(Path("./").glob("*.snap")) + except StopIteration: + exit("no snap found in ./.") + + logger.info(f"Found snap {snap_name}.") + + def _push_and_install(unitname, snapname): + try: + _push_to_unit(unitname, snapname) + _install_in_unit(unitname, snapname, clean=clean) + except SnapCtlError as e: + sys.exit(e.args[0]) + + if "/" in target: + logger.info(f'target is a unit. Pushing to {target}.') + + unit_name = target + _push_and_install(unit_name, snap_name) + else: + # todo parallelize + units = get_all_units(model, filter_apps=(target,)) + if not units: + exit(f'application {target} has no units. Is the app allocating?') + + logger.info(f'target is an app. Pushing to {units}.') + + for target_unit in units: + _push_and_install(target_unit.unit_name, snap_name) + + +def push_snap( + target: str = typer.Argument( + None, + help="Unit to which the snap should be pushed. If a unit ID is omitted (i.e. if you pass an " + "application name), this command will push the snap to all units.", + ), + snap: Path = typer.Option( + Path(os.getcwd()).absolute(), "--snap", "-s", + help="Root path of the snap package." + ), + rebuild: bool = typer.Option( + False, + "--rebuild", + "-r", + help="Whether to rebuild the snap and push the resulting build, or use an existing .snap file.", + ), + model: str = typer.Option( + None, + "--model", + "-m", + help="Model in which to find the target. Defaults to the current model.", + ), + clean: CleanOpt = typer.Option(CleanOpt.yes, "--clean", "-c", help=""), +): + """Install a local snap into a live machine charm.""" + return _push_snap( + target=target, + snap=snap, + rebuild=rebuild, + model=model, + clean=clean, + ) + + +if __name__ == '__main__': + _push_snap("zookeeper", snap=Path('/home/pietro/canonical/zookeeper-snap/'), rebuild=False) diff --git a/jhack/tests/utils/test_tail.py b/jhack/tests/utils/test_tail.py index dacb23f..0288910 100644 --- a/jhack/tests/utils/test_tail.py +++ b/jhack/tests/utils/test_tail.py @@ -7,7 +7,8 @@ import jhack.utils.tail_charms jhack.utils.tail_charms.MODEL_LOGLEVEL = "DEBUG" -from jhack.utils.tail_charms import Processor, Target, _tail_events +from jhack.helpers import Target +from jhack.utils.tail_charms import Processor, _tail_events def _mock_emit( diff --git a/jhack/utils/tail_charms.py b/jhack/utils/tail_charms.py index b048f9b..9d25e85 100644 --- a/jhack/utils/tail_charms.py +++ b/jhack/utils/tail_charms.py @@ -5,7 +5,6 @@ import time from collections import Counter from dataclasses import dataclass, field -from itertools import chain from pathlib import Path from typing import ( Callable, @@ -29,7 +28,7 @@ from rich.table import Column, Table from rich.text import Text -from jhack.helpers import JPopen, juju_status +from jhack.helpers import JPopen, Target, get_all_units from jhack.logger import logger as jhacklogger from jhack.utils.debug_log_interlacer import DebugLogInterlacer @@ -71,43 +70,6 @@ def model_loglevel(model: str = None): return "WARNING" # the default -@dataclass -class Target: - app: str - unit: int - leader: bool = False - - @staticmethod - def from_name(name: str): - if "/" not in name: - logger.warning( - "invalid target name: expected `/`; " - f"got {name!r}." - ) - app, unit_ = name.split("/") - leader = unit_.endswith("*") - unit = unit_.strip("*") - return Target(app, unit, leader=leader) - - @property - def unit_name(self): - return f"{self.app}/{self.unit}" - - def __hash__(self): - return hash((self.app, self.unit, self.leader)) - - -def get_all_units(model: str = None) -> Sequence[Target]: - status = juju_status(json=True, model=model) - # sub charms don't have units or applications - units = list( - chain( - *(app.get("units", ()) for app in status.get("applications", {}).values()) - ) - ) - return tuple(map(Target.from_name, units)) - - def parse_targets(targets: str = None, model: str = None) -> Sequence[Target]: if not targets: return get_all_units(model=model) From 2e63120784700301be0a3274c48b84921a44a384 Mon Sep 17 00:00:00 2001 From: Pietro Pasotti Date: Thu, 2 Mar 2023 15:59:09 +0100 Subject: [PATCH 2/2] vbump --- jhack/snap/push_snap.py | 146 +++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- snap/snapcraft.yaml | 2 +- 3 files changed, 131 insertions(+), 19 deletions(-) diff --git a/jhack/snap/push_snap.py b/jhack/snap/push_snap.py index ab11321..79728e1 100644 --- a/jhack/snap/push_snap.py +++ b/jhack/snap/push_snap.py @@ -1,10 +1,11 @@ import os +import re import sys from contextlib import contextmanager from enum import Enum from pathlib import Path from subprocess import CalledProcessError, run -from typing import Literal +from typing import Literal, List import typer @@ -26,10 +27,14 @@ class SnapCtlError(RuntimeError): pass -def _push_to_unit(unit_name: str, snap_name: str): +def _push_to_unit(unit_name: str, snap_name: str, dry_run: bool = False): logger.info(f"shelling snap over to {unit_name}...") cmd = f"juju scp ./{snap_name} {unit_name}:~/" - logger.debug("running: cmd") + logger.debug(f"running: {cmd}") + + if dry_run: + print(f"would shell over {snap_name} with {cmd!r}") + return def fail(): fail_msg = f"Failed scp'ing {snap_name} to {unit_name}" @@ -54,10 +59,15 @@ def fail(): logger.info("\tshelled.") -def _clean(unit_name, snap_name, mode): +def _clean(unit_name, snap_name, mode, dry_run: bool = False): _mode = " --purge" if mode == "force" else "" logger.info(f"cleaning up{_mode} {snap_name} in {unit_name}") cmd = f"juju ssh {unit_name} -- sudo snap remove {snap_name}{_mode}" + + if dry_run: + print(f"would clean {unit_name} with {cmd!r}") + return + # check=False as we don't care if this fails because e.g. the snap was not installed. out = JPopen(cmd.split(), wait=True).stdout.read().decode("utf-8") logger.debug(out) @@ -65,15 +75,20 @@ def _clean(unit_name, snap_name, mode): def _install_in_unit( - unit_name: str, snap_name: str, clean: Literal[False, True, "force"] + unit_name: str, snap_name: str, clean: Literal[False, True, "force"], + dry_run: bool = False, ): if clean: _clean(unit_name, snap_name, mode=clean) logger.info(f"installing snap in {unit_name}...") - cmd = f"juju ssh {unit_name} -- sudo snap install --dangerous '~/{snap_name}'" + cmd = f"juju ssh {unit_name} -- sudo snap install --dangerous ~/{snap_name}" logger.debug(f"command = {cmd}") + if dry_run: + print(f"would install {snap_name} in {unit_name} with {cmd!r}") + return + try: out = JPopen( cmd.split(), wait=True @@ -93,12 +108,90 @@ class CleanOpt(str, Enum): force = "force" +bind_re = re.compile(r"([\w-]+)/([\w-]+):([\w-]+)") + + +def _connect_in_unit(unitname: str, snapname: str, plugs: List[str], slots: List[str], + dry_run: bool = False): + def connect_bind(spec: str, is_slot: bool): + m = bind_re.match(spec) + if not m: + logger.error(f'spec {spec} is not a valid bind specification: expected format is: ' + f'/:') + return + + local_endpoint, remote_snap, remote_endpoint = m.groups() + local_mount = f"{snapname}:{local_endpoint}" + remote_mount = f"{remote_snap}:{remote_endpoint}" + + # snap connect syntax is: + # PLUG SLOT + if is_slot: + # local side is the slot aka RIGHT side: + a, b = remote_mount, local_mount + + else: + a, b = local_mount, remote_mount + + cmd = f"juju ssh {unitname} -- sudo snap connect {a} {b}" + + logger.debug(f"running: {cmd}") + if dry_run: + print(f'would have connected {a} --> {b} by running: {cmd!r}') + return + + try: + proc = JPopen(cmd.split(), wait=True) + except (FileNotFoundError, CalledProcessError) as e: + logger.error(exc_info=True) + raise SnapCtlError(f"cmd {cmd} failed.") from e + + proc.wait() + out = proc.stdout.read().decode("utf-8") + + if proc.returncode != 0: + err = proc.stderr.read().decode("utf-8") + logger.error(f'stderr = {err}') + fail_msg = f"Failed snap-connecting {a} to {b}" + logger.error(fail_msg, exc_info=True) + else: + logger.debug(out) + logger.info("\tshelled.") + + logger.info('binding slots and plugs...') + for spec in (slots or ()): + connect_bind(spec, is_slot=True) + + for spec in (plugs or ()): + connect_bind(spec, is_slot=False) + logger.info('\tAll bound.') + + +def _rebuild(dry_run: bool): + """Run snapcraft in current dir.""" + logger.info("snapping...") + cmd = "snapcraft --use-lxd" + + if dry_run: + print(f'would call {cmd!r}') + return + + try: + run(cmd.split(), capture_output=True, check=True) + except CalledProcessError: + logger.error('failed packing snap.', exc_info=True) + exit("failed packing snap.") + + def _push_snap( target: str, snap: Path = Path("./"), rebuild: bool = False, model: str = None, clean: CleanOpt = CleanOpt.yes, + bind_slots: List[str] = None, + bind_plugs: List[str] = None, + dry_run: bool = False, ): if get_substrate(model) == "k8s": exit(f"{model or 'this model'} is not a machine model.") @@ -106,24 +199,26 @@ def _push_snap( logger.info(f"snap root={snap}") with cwd(snap): if rebuild: - logger.info("snapping...") - try: - run(["snapcraft", "--use-lxd"], capture_output=True, check=True) - except CalledProcessError: - logger.error('failed packing snap.', exc_info=True) - exit("failed packing snap.") + _rebuild(dry_run=dry_run) try: snap_name = next(Path("./").glob("*.snap")) except StopIteration: - exit("no snap found in ./.") + if dry_run: + snap_name = "my-application.dry-run.snap" + print(f"no snap found in current folder; assuming at this point " + f"you will have something like {snap_name} in {snap.absolute()}") + + else: + exit("no snap found in ./.") logger.info(f"Found snap {snap_name}.") def _push_and_install(unitname, snapname): try: - _push_to_unit(unitname, snapname) - _install_in_unit(unitname, snapname, clean=clean) + _push_to_unit(unitname, snapname, dry_run=dry_run) + _install_in_unit(unitname, snapname, clean=clean, dry_run=dry_run) + _connect_in_unit(unitname, snapname, plugs=bind_plugs, slots=bind_slots, dry_run=dry_run) except SnapCtlError as e: sys.exit(e.args[0]) @@ -154,6 +249,16 @@ def push_snap( Path(os.getcwd()).absolute(), "--snap", "-s", help="Root path of the snap package." ), + bind_slots: List[str] = typer.Option( + None, "--bind-slots", "-b", + help="List of `/:` definitions. " + "Will be connected after the snap is installed." + ), + bind_plugs: List[str] = typer.Option( + None, "--bind-plugs", "-p", + help="List of `/:` definitions. " + "Will be connected after the snap is installed." + ), rebuild: bool = typer.Option( False, "--rebuild", @@ -166,7 +271,11 @@ def push_snap( "-m", help="Model in which to find the target. Defaults to the current model.", ), - clean: CleanOpt = typer.Option(CleanOpt.yes, "--clean", "-c", help=""), + clean: CleanOpt = typer.Option( + CleanOpt.yes, "--clean", "-c", + help="Uninstall the existing snap installation before installing the newly pushed snap."), + dry_run: bool = typer.Option(False, "--dry-run", + help="Do nothing, show what would have happened."), ): """Install a local snap into a live machine charm.""" return _push_snap( @@ -175,8 +284,11 @@ def push_snap( rebuild=rebuild, model=model, clean=clean, + bind_slots=bind_slots, + bind_plugs=bind_plugs, + dry_run=dry_run ) if __name__ == '__main__': - _push_snap("zookeeper", snap=Path('/home/pietro/canonical/zookeeper-snap/'), rebuild=False) + _push_snap("zoo/0", snap=Path('/home/pietro/canonical/zookeeper-snap/'), rebuild=False) diff --git a/pyproject.toml b/pyproject.toml index 14a0dab..b3061b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "jhack" # TODO: keep snapcraft.yaml in sync with version! -version = "0.3.14" +version = "0.3.15" authors = [ { name = "Pietro Pasotti", email = "pietro.pasotti@canonical.com" } ] diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index da0c30b..6e96ad0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -10,7 +10,7 @@ architectures: - build-on: i386 # TODO: keep pyproject in sync with version! -version: '0.3.14' +version: '0.3.15' summary: Cli tool packed with juju hacks. description: |