From 0ac1f9d2ced6631b8eebb4b84e1271fc1a87602d Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 13 Nov 2024 15:11:30 -0800 Subject: [PATCH 1/4] add threading to docker compose run --- devservices/utils/docker_compose.py | 39 +++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/devservices/utils/docker_compose.py b/devservices/utils/docker_compose.py index 75b97f0..ca0a65e 100644 --- a/devservices/utils/docker_compose.py +++ b/devservices/utils/docker_compose.py @@ -1,5 +1,6 @@ from __future__ import annotations +import concurrent import logging import os import platform @@ -222,6 +223,20 @@ def _get_docker_compose_commands_to_run( return docker_compose_commands +def _run_cmd(cmd: list[str], env: dict[str, str]) -> subprocess.CompletedProcess[str]: + logger = logging.getLogger(LOGGER_NAME) + try: + logger.debug(f"Running command: {' '.join(cmd)}") + return subprocess.run(cmd, check=True, capture_output=True, text=True, env=env) + except subprocess.CalledProcessError as e: + raise DockerComposeError( + command=" ".join(cmd), + returncode=e.returncode, + stdout=e.stdout, + stderr=e.stderr, + ) from e + + def run_docker_compose_command( service: Service, command: str, @@ -252,21 +267,13 @@ def run_docker_compose_command( ) cmd_outputs = [] - for cmd in docker_compose_commands: - try: - logger = logging.getLogger(LOGGER_NAME) - logger.debug(f"Running command: {' '.join(cmd)}") - cmd_outputs.append( - subprocess.run( - cmd, check=True, capture_output=True, text=True, env=current_env - ) - ) - except subprocess.CalledProcessError as e: - raise DockerComposeError( - command=command, - returncode=e.returncode, - stdout=e.stdout, - stderr=e.stderr, - ) from e + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [ + executor.submit(_run_cmd, cmd, current_env) + for cmd in docker_compose_commands + ] + for future in concurrent.futures.as_completed(futures): + cmd_outputs.append(future.result()) return cmd_outputs From 0028b6685ca622ae084de0091d6beac907dd2bc6 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 13 Nov 2024 15:16:42 -0800 Subject: [PATCH 2/4] use passed in command --- devservices/utils/docker_compose.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/devservices/utils/docker_compose.py b/devservices/utils/docker_compose.py index ca0a65e..a30312b 100644 --- a/devservices/utils/docker_compose.py +++ b/devservices/utils/docker_compose.py @@ -223,14 +223,16 @@ def _get_docker_compose_commands_to_run( return docker_compose_commands -def _run_cmd(cmd: list[str], env: dict[str, str]) -> subprocess.CompletedProcess[str]: +def _run_cmd( + cmd: list[str], env: dict[str, str], command: str +) -> subprocess.CompletedProcess[str]: logger = logging.getLogger(LOGGER_NAME) try: logger.debug(f"Running command: {' '.join(cmd)}") return subprocess.run(cmd, check=True, capture_output=True, text=True, env=env) except subprocess.CalledProcessError as e: raise DockerComposeError( - command=" ".join(cmd), + command=command, returncode=e.returncode, stdout=e.stdout, stderr=e.stderr, @@ -270,7 +272,7 @@ def run_docker_compose_command( with concurrent.futures.ThreadPoolExecutor() as executor: futures = [ - executor.submit(_run_cmd, cmd, current_env) + executor.submit(_run_cmd, cmd, current_env, command) for cmd in docker_compose_commands ] for future in concurrent.futures.as_completed(futures): From 245d4f233eb7f00277cbe83232b7237e2df3ed77 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 13 Nov 2024 15:36:14 -0800 Subject: [PATCH 3/4] add wait for healthy services when starting --- devservices/commands/start.py | 2 +- tests/commands/test_start.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/devservices/commands/start.py b/devservices/commands/start.py index 3184607..8a493ff 100644 --- a/devservices/commands/start.py +++ b/devservices/commands/start.py @@ -60,7 +60,7 @@ def start(args: Namespace) -> None: "up", mode_dependencies, remote_dependencies, - options=["-d"], + options=["-d", "--wait"], ) except DockerComposeError as dce: status.failure(f"Failed to start {service.name}: {dce.stderr}") diff --git a/tests/commands/test_start.py b/tests/commands/test_start.py index 899ee1e..45f6365 100644 --- a/tests/commands/test_start.py +++ b/tests/commands/test_start.py @@ -78,6 +78,7 @@ def test_start_simple( "clickhouse", "redis", "-d", + "--wait", ], check=True, capture_output=True, From c4c74bacc4573c0bfc4b85c911297f2b3829133d Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 13 Nov 2024 16:00:31 -0800 Subject: [PATCH 4/4] add more context to docker compose error --- devservices/utils/docker_compose.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/devservices/utils/docker_compose.py b/devservices/utils/docker_compose.py index a30312b..ca0a65e 100644 --- a/devservices/utils/docker_compose.py +++ b/devservices/utils/docker_compose.py @@ -223,16 +223,14 @@ def _get_docker_compose_commands_to_run( return docker_compose_commands -def _run_cmd( - cmd: list[str], env: dict[str, str], command: str -) -> subprocess.CompletedProcess[str]: +def _run_cmd(cmd: list[str], env: dict[str, str]) -> subprocess.CompletedProcess[str]: logger = logging.getLogger(LOGGER_NAME) try: logger.debug(f"Running command: {' '.join(cmd)}") return subprocess.run(cmd, check=True, capture_output=True, text=True, env=env) except subprocess.CalledProcessError as e: raise DockerComposeError( - command=command, + command=" ".join(cmd), returncode=e.returncode, stdout=e.stdout, stderr=e.stderr, @@ -272,7 +270,7 @@ def run_docker_compose_command( with concurrent.futures.ThreadPoolExecutor() as executor: futures = [ - executor.submit(_run_cmd, cmd, current_env, command) + executor.submit(_run_cmd, cmd, current_env) for cmd in docker_compose_commands ] for future in concurrent.futures.as_completed(futures):