feat(devservices): Run commands for nested dependencies by project

devservices/commands/logs.py

@@ -33,12 +33,13 @@ def logs(args: Namespace) -> None:
     modes = service.config.modes
     # TODO: allow custom modes to be used
     mode_to_use = "default"
-    mode_dependencies = " ".join(modes[mode_to_use])
+    mode_dependencies = modes[mode_to_use]
 
     try:
-        logs = run_docker_compose_command(service, f"logs {mode_dependencies}")
+        logs = run_docker_compose_command(service, "logs", mode_dependencies)
     except DockerComposeError as dce:
         print(f"Failed to get logs for {service.name}: {dce.stderr}")
         exit(1)
-    sys.stdout.write(logs.stdout)
-    sys.stdout.flush()
+    for log in logs:
+        sys.stdout.write(log.stdout)
+        sys.stdout.flush()

devservices/commands/start.py

@@ -30,12 +30,12 @@ def start(args: Namespace) -> None:
     modes = service.config.modes
     # TODO: allow custom modes to be used
     mode_to_start = "default"
-    mode_dependencies = " ".join(modes[mode_to_start])
+    mode_dependencies = modes[mode_to_start]
 
     with Status(f"Starting {service.name}", f"{service.name} started") as status:
         try:
             run_docker_compose_command(
-                service, f"up -d {mode_dependencies}", force_update_dependencies=True
+                service, "up", mode_dependencies, ["-d"], force_update_dependencies=True
             )
         except DockerComposeError as dce:
             status.print(f"Failed to start {service.name}: {dce.stderr}")

devservices/commands/status.py

@@ -68,21 +68,22 @@ def status(args: Namespace) -> None:
     modes = service.config.modes
     # TODO: allow custom modes to be used
     mode_to_view = "default"
-    mode_dependencies = " ".join(modes[mode_to_view])
+    mode_dependencies = modes[mode_to_view]
 
     try:
-        status_json = run_docker_compose_command(
-            service, f"ps {mode_dependencies} --format json"
-        ).stdout
+        status_jsons = run_docker_compose_command(
+            service, "ps", mode_dependencies, options=["--format", "json"]
+        )
     except DockerComposeError as dce:
         print(f"Failed to get status for {service.name}: {dce.stderr}")
         exit(1)
     # If the service is not running, the status_json will be empty
-    if not status_json:
+    if len(status_jsons) == 0:
         print(f"{service.name} is not running")
         return
     output = f"Service: {service.name}\n\n"
-    output += format_status_output(status_json)
+    for status_json in status_jsons:
+        output += format_status_output(status_json.stdout)
     output += "=" * LINE_LENGTH
     sys.stdout.write(output + "\n")
     sys.stdout.flush()

devservices/commands/stop.py

@@ -30,11 +30,11 @@ def stop(args: Namespace) -> None:
     modes = service.config.modes
     # TODO: allow custom modes to be used
     mode_to_stop = "default"
-    mode_dependencies = " ".join(modes[mode_to_stop])
+    mode_dependencies = modes[mode_to_stop]
 
     with Status(f"Stopping {service.name}", f"{service.name} stopped") as status:
         try:
-            run_docker_compose_command(service, f"down {mode_dependencies}")
+            run_docker_compose_command(service, "down", mode_dependencies)
         except DockerComposeError as dce:
             status.print(f"Failed to stop {service.name}: {dce.stderr}")
             exit(1)

devservices/configs/service_config.py

@@ -20,6 +20,7 @@ class RemoteConfig:
     repo_name: str
     branch: str
     repo_link: str
+    mode: str = "default"
 
 
 @dataclass

devservices/utils/dependencies.py

@@ -32,6 +32,7 @@
 class InstalledRemoteDependency:
     service_name: str
     repo_path: str
+    mode: str = "default"
 
 
 class SparseCheckoutManager:
@@ -160,7 +161,9 @@ def get_installed_remote_dependencies(
             ) from e
         installed_dependencies.add(
             InstalledRemoteDependency(
-                service_name=service_config.service_name, repo_path=dependency_repo_dir
+                service_name=service_config.service_name,
+                repo_path=dependency_repo_dir,
+                mode=remote_config.mode,
             )
         )
         nested_remote_configs = _get_remote_configs(
@@ -252,6 +255,7 @@ def install_dependency(dependency: RemoteConfig) -> set[InstalledRemoteDependenc
             InstalledRemoteDependency(
                 service_name=installed_config.service_name,
                 repo_path=dependency_repo_dir,
+                mode=dependency.mode,
             )
         ]
     )

devservices/utils/docker_compose.py

@@ -4,10 +4,12 @@
 import platform
 import re
 import subprocess
+from collections.abc import Callable
 from typing import cast
 
 from packaging import version
 
+from devservices.configs.service_config import load_service_config_from_file
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
@@ -21,6 +23,7 @@
 from devservices.exceptions import DockerComposeInstallationError
 from devservices.utils.dependencies import get_installed_remote_dependencies
 from devservices.utils.dependencies import install_dependencies
+from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.dependencies import verify_local_dependencies
 from devservices.utils.install_binary import install_binary
 from devservices.utils.services import Service
@@ -148,20 +151,103 @@
     install_docker_compose()
 
 
+# TODO: Consider removing this in favor of in house logic for determining non-remote services
+def _get_non_remote_services(
+    service_config_path: str, current_env: dict[str, str]
+) -> set[str]:
+    config_command = [
+        "docker",
+        "compose",
+        "-f",
+        service_config_path,
+        "config",
+        "--services",
+    ]
+    try:
+        config_services = subprocess.run(
+            config_command, capture_output=True, text=True, env=current_env
+        ).stdout
+    except subprocess.CalledProcessError as e:
+        raise DockerComposeError(
+            command=" ".join(config_command),
+            returncode=e.returncode,
+            stdout=e.stdout,
+            stderr=e.stderr,
+        ) from e
+    services_defined_in_config = set(config_services.splitlines())
+    return services_defined_in_config
+
+
+def _get_docker_compose_commands_to_run(
+    service: Service,
+    remote_dependencies: set[InstalledRemoteDependency],
+    current_env: dict[str, str],
+    command: str,
+    options: list[str],
+    service_config_file_path: str,
+    mode_dependencies: list[str],
+) -> list[list[str]]:
+    docker_compose_commands = []
+    create_docker_compose_command: Callable[[str, str, set[str]], list[str]] = (
+        lambda name, config_path, services_to_use: [
+            "docker",
+            "compose",
+            "-p",
+            name,
+            "-f",
+            config_path,
+            command,
+        ]
+        + sorted(list(services_to_use))  # Sort the services to prevent flaky tests
+        + options
+    )
+    # Sort the remote dependencies by service name to ensure a deterministic order
+    for dependency in sorted(remote_dependencies, key=lambda x: x.service_name):
+        dependency_service_config = load_service_config_from_file(dependency.repo_path)
+        dependency_config_path = os.path.join(
+            dependency.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
+        )
+        services_defined = _get_non_remote_services(dependency_config_path, current_env)
+        services_to_use = services_defined.intersection(
+            set(dependency_service_config.modes[dependency.mode])
+        )
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                dependency_service_config.service_name,
+                dependency_config_path,
+                services_to_use,
+            )
+        )
+
+    # Add docker compose command for the top level service
+    services_defined = _get_non_remote_services(service_config_file_path, current_env)
+    services_to_use = services_defined.intersection(set(mode_dependencies))
+    docker_compose_commands.append(
+        create_docker_compose_command(
+            service.name, service_config_file_path, services_to_use
+        )
+    )
+    return docker_compose_commands
+
+
 def run_docker_compose_command(
-    service: Service, command: str, force_update_dependencies: bool = False
-) -> subprocess.CompletedProcess[str]:
+    service: Service,
+    command: str,
+    mode_dependencies: list[str],
+    options: list[str] = [],
+    force_update_dependencies: bool = False,
+) -> list[subprocess.CompletedProcess[str]]:
     dependencies = list(service.config.dependencies.values())
     if force_update_dependencies:
-        install_dependencies(dependencies)
+        remote_dependencies = install_dependencies(dependencies)
     else:
         are_dependencies_valid = verify_local_dependencies(dependencies)
         if not are_dependencies_valid:
             # TODO: Figure out how to handle this case as installing dependencies may not be the right thing to do
             #       since the dependencies may have changed since the service was started.
-            install_dependencies(dependencies)
+            remote_dependencies = install_dependencies(dependencies)
         else:
-            get_installed_remote_dependencies(dependencies)
+            remote_dependencies = get_installed_remote_dependencies(dependencies)
     relative_local_dependency_directory = os.path.relpath(
         os.path.join(DEVSERVICES_DEPENDENCIES_CACHE_DIR, DEPENDENCY_CONFIG_VERSION),
         service.repo_path,
@@ -174,26 +260,30 @@
     current_env[
         DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
     ] = relative_local_dependency_directory
-    cmd = [
-        "docker",
-        "compose",
-        "-p",
-        service.name,
-        "-f",
-        service_config_file_path,
-    ] + command.split()
-    try:
-        return subprocess.run(
-            cmd,
-            check=True,
-            capture_output=True,
-            text=True,
-            env=current_env,
-        )
-    except subprocess.CalledProcessError as e:
-        raise DockerComposeError(
-            command=command,
-            returncode=e.returncode,
-            stdout=e.stdout,
-            stderr=e.stderr,
-        ) from e
+    docker_compose_commands = _get_docker_compose_commands_to_run(
+        service=service,
+        remote_dependencies=remote_dependencies,
+        current_env=current_env,
+        command=command,
+        options=options,
+        service_config_file_path=service_config_file_path,
+        mode_dependencies=mode_dependencies,
+    )
+
+    cmd_outputs = []
+    for cmd in docker_compose_commands:
+        try:
+            cmd_outputs.append(
+                subprocess.run(
+                    cmd, check=True, capture_output=True, text=True, env=current_env
+                )
+            )
+        except subprocess.CalledProcessError as e:
+            raise DockerComposeError(
+                command=command,
+                returncode=e.returncode,
+                stdout=e.stdout,
+                stderr=e.stderr,
+            ) from e
+
+    return cmd_outputs

testing/resources/child-service-repo/README.md

@@ -0,0 +1 @@
+This is a child service that is used for testing purposes.

testing/resources/child-service-repo/devservices/config.yml

@@ -0,0 +1,19 @@
+x-sentry-service-config:
+  version: 0.1
+  service_name: child-service
+  dependencies:
+    child-service:
+      description: This is a remote child service that is used for testing purposes.
+  modes:
+    default: [child-service]
+
+services:
+  child-service:
+    image: child-service
+    networks:
+      - sentry
+
+networks:
+  sentry:
+    name: sentry
+    external: true

testing/resources/grandparent-service-repo/README.md

@@ -0,0 +1 @@
+This is a grandparent service that is used for testing purposes.

testing/resources/grandparent-service-repo/devservices/config.yml

@@ -0,0 +1,24 @@
+x-sentry-service-config:
+  version: 0.1
+  service_name: grandparent-service
+  dependencies:
+    parent-service:
+      description: This is a remote parent service that is used for testing purposes.
+      remote:
+        repo_name: parent-service
+        branch: main
+        repo_link: https://github.com/example/parent-service.git
+    grandparent-service:
+      description: This is a remote nested dependency service that is used for testing purposes.
+  modes:
+    default: [parent-service, grandparent-service]
+services:
+  grandparent-service:
+    image: grandparent-service
+    networks:
+      - sentry
+
+networks:
+  sentry:
+    name: sentry
+    external: true

testing/resources/parent-service-repo/README.md

@@ -0,0 +1 @@
+This is a parent service that is used for testing purposes.

testing/resources/parent-service-repo/devservices/config.yml

@@ -0,0 +1,24 @@
+x-sentry-service-config:
+  version: 0.1
+  service_name: parent-service
+  dependencies:
+    child-service:
+      description: This is a remote child service that is used for testing purposes.
+      remote:
+        repo_name: child-service
+        branch: main
+        repo_link: https://github.com/example/child-service.git
+    parent-service:
+      description: This is a remote parent service that is used for testing purposes.
+  modes:
+    default: [parent-service, child-service]
+services:
+  parent-service:
+    image: parent-service
+    networks:
+      - sentry
+
+networks:
+  sentry:
+    name: sentry
+    external: true