From 580e40ab08b21e5c95252e09ce9b51330b197ba2 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 10:38:09 -0800 Subject: [PATCH 01/19] wip: dind --- latch_cli/constants.py | 2 +- latch_cli/main.py | 23 +++++++++-------------- latch_cli/services/init/init.py | 30 ++++++++++++++++++++++-------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/latch_cli/constants.py b/latch_cli/constants.py index 1b885c63..7f2fb3a0 100644 --- a/latch_cli/constants.py +++ b/latch_cli/constants.py @@ -7,7 +7,7 @@ @dataclass(frozen=True) class LatchConstants: base_image: str = ( - "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:ace9-main" + "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:0d3d-main" ) mib: int = 2**20 diff --git a/latch_cli/main.py b/latch_cli/main.py index b4c244e9..9055bb89 100644 --- a/latch_cli/main.py +++ b/latch_cli/main.py @@ -10,7 +10,7 @@ import latch_cli.click_utils from latch_cli.exceptions.handler import CrashHandler -from latch_cli.services.init.init import template_flag_to_option +from latch_cli.services.init.init import template_flag_to_option, BaseImageOptions from latch_cli.utils import get_latest_package_version, get_local_package_version latch_cli.click_utils.patch() @@ -161,23 +161,18 @@ def login(connection: Optional[str]): default=False, ) @click.option( - "--cuda", - help="Create a user editable Dockerfile for this workflow.", - is_flag=True, - default=False, -) -@click.option( - "--opencl", - help="Create a user editable Dockerfile for this workflow.", - is_flag=True, - default=False, + "--base-image", + help="Which base image to use for the Dockerfile.", + type=click.Choice( + list(BaseImageOptions._member_names_), + case_sensitive=False, + ) ) def init( pkg_name: str, template: Optional[str] = None, dockerfile: bool = False, - cuda: bool = False, - opencl: bool = False, + base_image: str = "default", ): """Initialize boilerplate for local workflow code.""" @@ -186,7 +181,7 @@ def init( from latch_cli.services.init import init - created = init(pkg_name, template, dockerfile, cuda, opencl) + created = init(pkg_name, template, dockerfile, base_image) if created: click.secho(f"Created a latch workflow in `{pkg_name}`", fg="green") click.secho("Run", fg="green") diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index c8ab63d0..6c1409b5 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -8,6 +8,7 @@ from datetime import datetime from pathlib import Path from typing import Callable, Optional +from enum import Enum import click from pkg_resources import get_distribution @@ -140,29 +141,46 @@ def _gen_example_conda(pkg_root: Path): conda_env_src = source_path / "environment.yaml" shutil.copy(conda_env_src, conda_env_dest) +def _get_example_docker(pkg_root: Path): + pkg_root = pkg_root.resolve() + source_path = Path(__file__).parent / "example_docker" + + _get_boilerplate(pkg_root, source_path) + + conda_env_dest = pkg_root / "environment.yaml" + conda_env_src = source_path / "environment.yaml" + shutil.copy(conda_env_src, conda_env_dest) + option_map = { "Empty workflow": _gen_template, "Subprocess Example": _gen_assemble_and_sort, "R Example": _gen_example_r, "Conda Example": _gen_example_conda, + "Docker Example": _get_example_docker, } template_flag_to_option = { "empty": "Empty workflow", + "docker": "Docker Example", "subprocess": "Subprocess Example", "r": "R Example", "conda": "Conda Example", } +class BaseImageOptions(Enum): + cuda = "cuda" + opencl = "opencl" + docker = "docker" + + def init( pkg_name: str, template: Optional[str], expose_dockerfile: bool = True, - cuda: bool = False, - opencl: bool = False, + base_image_type: Optional[str] = None, ) -> bool: """Creates boilerplate workflow files in the user's working directory. @@ -261,12 +279,8 @@ def init( return False base_image = latch_constants.base_image - if cuda and opencl: - raise ValueError("Latch does not support both CUDA and OpenCL yet") - elif cuda: - base_image = base_image.replace("latch-base", "latch-base-cuda") - elif opencl: - base_image = base_image.replace("latch-base", "latch-base-opencl") + if base_image_type is not None: + base_image = base_image.replace("latch-base", f"latch-base-{base_image_type}") config = LatchWorkflowConfig( latch_version=get_distribution("latch").version, From b609309ffbbd9faa2b888036f0aae7da1e2abd6f Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 11:42:23 -0800 Subject: [PATCH 02/19] fix: sysbox pod defs --- latch/resources/tasks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/latch/resources/tasks.py b/latch/resources/tasks.py index d135a37e..54e66371 100644 --- a/latch/resources/tasks.py +++ b/latch/resources/tasks.py @@ -87,7 +87,9 @@ def _get_large_pod() -> Pod: primary_container.resources = resources return Pod( + annotations={"io.kubernetes.cri-o.userns-mode": "auto:size=65536"}, pod_spec=V1PodSpec( + runtime_class_name="sysbox-runc", containers=[primary_container], tolerations=[ V1Toleration(effect="NoSchedule", key="ng", value="cpu-96-spot") @@ -108,7 +110,9 @@ def _get_medium_pod() -> Pod: primary_container.resources = resources return Pod( + annotations={"io.kubernetes.cri-o.userns-mode": "auto:size=65536"}, pod_spec=V1PodSpec( + runtime_class_name="sysbox-runc", containers=[primary_container], tolerations=[ V1Toleration(effect="NoSchedule", key="ng", value="cpu-32-spot") @@ -129,7 +133,9 @@ def _get_small_pod() -> Pod: primary_container.resources = resources return Pod( + annotations={"io.kubernetes.cri-o.userns-mode": "auto:size=65536"}, pod_spec=V1PodSpec( + runtime_class_name="sysbox-runc", containers=[primary_container], ), primary_container_name="primary", @@ -289,14 +295,18 @@ def custom_task(cpu: int, memory: int): primary_container.resources = resources if cpu < 48 and memory < 128: task_config = Pod( + annotations={"io.kubernetes.cri-o.userns-mode": "auto:size=65536"}, pod_spec=V1PodSpec( + runtime_class_name="sysbox-runc", containers=[primary_container], ), primary_container_name="primary", ) elif cpu < 96 and memory < 180: task_config = Pod( + annotations={"io.kubernetes.cri-o.userns-mode": "auto:size=65536"}, pod_spec=V1PodSpec( + runtime_class_name="sysbox-runc", containers=[primary_container], tolerations=[ V1Toleration(effect="NoSchedule", key="ng", value="cpu-96-spot") From d6b38394beb7bbf56968c092ab39e256e2155604 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 15:02:10 -0800 Subject: [PATCH 03/19] fix: example for docker --- latch_cli/constants.py | 2 +- latch_cli/main.py | 4 +- .../services/init/example_docker/assemble.py | 65 +++++++++++++++++++ latch_cli/services/init/init.py | 44 +++++++++---- 4 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 latch_cli/services/init/example_docker/assemble.py diff --git a/latch_cli/constants.py b/latch_cli/constants.py index 7f2fb3a0..a29de932 100644 --- a/latch_cli/constants.py +++ b/latch_cli/constants.py @@ -7,7 +7,7 @@ @dataclass(frozen=True) class LatchConstants: base_image: str = ( - "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:0d3d-main" + "812206152185.dkr.ecr.us-west-2.amazonaws.com/latch-base:9c8f-main" ) mib: int = 2**20 diff --git a/latch_cli/main.py b/latch_cli/main.py index 9055bb89..40a5142f 100644 --- a/latch_cli/main.py +++ b/latch_cli/main.py @@ -10,7 +10,7 @@ import latch_cli.click_utils from latch_cli.exceptions.handler import CrashHandler -from latch_cli.services.init.init import template_flag_to_option, BaseImageOptions +from latch_cli.services.init.init import BaseImageOptions, template_flag_to_option from latch_cli.utils import get_latest_package_version, get_local_package_version latch_cli.click_utils.patch() @@ -166,7 +166,7 @@ def login(connection: Optional[str]): type=click.Choice( list(BaseImageOptions._member_names_), case_sensitive=False, - ) + ), ) def init( pkg_name: str, diff --git a/latch_cli/services/init/example_docker/assemble.py b/latch_cli/services/init/example_docker/assemble.py new file mode 100644 index 00000000..cea2fa71 --- /dev/null +++ b/latch_cli/services/init/example_docker/assemble.py @@ -0,0 +1,65 @@ +import subprocess +from pathlib import Path + +from latch import small_task +from latch.functions.messages import message +from latch.types import LatchFile, LatchOutputDir + + +@small_task +def assembly_task( + read1: LatchFile, read2: LatchFile, output_directory: LatchOutputDir +) -> LatchFile: + + outdir = Path("/root/outputs").resolve() + outdir.mkdir(exist_ok=True) + + bowtie2_cmd = [ + "docker", + "run", + "--user", + "root", + "--env", + "BOWTIE2_INDEXES=/reference", + "--mount", + "type=bind,source=/root/reference,target=/reference", + "--mount", + f"type=bind,source={read1.local_path},target=/r1.fq", + "--mount", + f"type=bind,source={read2.local_path},target=/r2.fq", + "--mount", + f"type=bind,source={outdir},target=/outputs", + "biocontainers/bowtie2:v2.4.1_cv1", + "bowtie2", + "--local", + "--very-sensitive-local", + "-x", + "wuhan", + "-1", + "/r1.fq", + "-2", + "/r2.fq", + "-S", + "/outputs/covid_assembly.sam", + ] + + try: + # We use shell=True for all the benefits of pipes and other shell features. + # When using shell=True, we pass the entire command as a single string as + # opposed to a list since the shell will parse the string into a list + # using its own rules. + subprocess.run(" ".join(bowtie2_cmd), shell=True, check=True) + except subprocess.CalledProcessError as e: + # will display in the messages tab of the execution graph for the assembly_task node + message( + "error", + {"title": "Bowtie2 Failed", "body": f"Error: {str(e)}"}, + ) + raise e + + # intended output path of the file in Latch console, constructed from + # the user provided output directory + output_location = f"{output_directory.remote_directory}/covid_assembly.sam" + local_sam_file = outdir / "covid_assembly.sam" + + return LatchFile(str(local_sam_file), output_location) diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index 6c1409b5..0ac8c5f8 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -6,9 +6,9 @@ import subprocess from dataclasses import asdict from datetime import datetime +from enum import Enum from pathlib import Path from typing import Callable, Optional -from enum import Enum import click from pkg_resources import get_distribution @@ -53,15 +53,12 @@ def _get_boilerplate(pkg_root: Path, source_path: Path): f.write("0.0.0") -def _gen_assemble_and_sort(pkg_root: Path): +def _get_example_reference(pkg_root: Path): import boto3 from botocore import UNSIGNED from botocore.config import Config pkg_root = pkg_root.resolve() - source_path = Path(__file__).parent / "assemble_and_sort" - - _get_boilerplate(pkg_root, source_path) data_root = pkg_root / "reference" data_root.mkdir(exist_ok=True) @@ -85,6 +82,19 @@ def _gen_assemble_and_sort(pkg_root: Path): s3.download_fileobj("latch-public", f"sdk/{id}", f) print() + +def _gen_assemble_and_sort(pkg_root: Path): + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + + pkg_root = pkg_root.resolve() + source_path = Path(__file__).parent / "assemble_and_sort" + + _get_boilerplate(pkg_root, source_path) + + _get_example_reference(pkg_root) + print("Downloading bowtie2") bowtie2_base_name = "bowtie2-2.5.1-linux-x86_64" subprocess.run( @@ -141,15 +151,24 @@ def _gen_example_conda(pkg_root: Path): conda_env_src = source_path / "environment.yaml" shutil.copy(conda_env_src, conda_env_dest) -def _get_example_docker(pkg_root: Path): + +def _gen_example_docker(pkg_root: Path): + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + pkg_root = pkg_root.resolve() - source_path = Path(__file__).parent / "example_docker" + source_path = Path(__file__).parent / "assemble_and_sort" _get_boilerplate(pkg_root, source_path) + _get_example_reference(pkg_root) - conda_env_dest = pkg_root / "environment.yaml" - conda_env_src = source_path / "environment.yaml" - shutil.copy(conda_env_src, conda_env_dest) + source_docker_path = Path(__file__).parent / "example_docker" + shutil.copy(source_docker_path / "assemble.py", pkg_root / "wf/assemble.py") + + (pkg_root / ".env").unlink() + + print() option_map = { @@ -157,7 +176,7 @@ def _get_example_docker(pkg_root: Path): "Subprocess Example": _gen_assemble_and_sort, "R Example": _gen_example_r, "Conda Example": _gen_example_conda, - "Docker Example": _get_example_docker, + "Docker Example": _gen_example_docker, } @@ -282,6 +301,9 @@ def init( if base_image_type is not None: base_image = base_image.replace("latch-base", f"latch-base-{base_image_type}") + if template == "docker": + base_image = base_image.replace("latch-base", "latch-base-docker") + config = LatchWorkflowConfig( latch_version=get_distribution("latch").version, base_image=base_image, From ddcb9fa4582544dd6db2be58b375a8981f1b54a2 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 15:12:48 -0800 Subject: [PATCH 04/19] fix: update docs a bit --- docs/source/basics/defining_environment.md | 2 +- docs/source/subcommands.md | 8 ++------ latch_cli/main.py | 1 + 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index 15f0d992..2e96454b 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -4,7 +4,7 @@ Workflow code is rarely free of dependencies. It may require python or system pa Latch manages the execution environment of a workflow using Dockerfiles. A Dockerfile specifies the environment in which the workflow runs. -Latch has three [base images](https://docs.docker.com/build/building/base-images/), one without hardware acceleration drivers, one with CUDA drivers, and one with OPENCL drivers. To use the CUDA or OPENCL base image, use the `--cuda` or `--opencl` flags when running `latch init`. +Latch has four [base images](https://docs.docker.com/build/building/base-images/), one bare minimum, one with CUDA drivers, one with OPENCL drivers, and one with the Docker software pre-installed. To use the CUDA, OPENCL, or Docker base image, use the `--base-image` flag when running `latch init`. The workflow environment is encapsulated in [a Docker container](https://en.wikipedia.org/wiki/Docker_(software)), which is created from a recipe defined in [a text document called Dockerfile.](https://docs.docker.com/engine/reference/builder/) In most cases these recipes are repetitive and unnecessarily complicated, so Latch will automatically generate one using conventional dependency lists and heuristics. To use a handwritten Dockerfile, [run the eject command](#ejecting-auto-generation). diff --git a/docs/source/subcommands.md b/docs/source/subcommands.md index e86cc0be..02df81d8 100644 --- a/docs/source/subcommands.md +++ b/docs/source/subcommands.md @@ -20,13 +20,9 @@ One of `r`, `conda`, `subprocess`, `empty`. If not provided, user will be prompt Generate a Dockerfile for the workflow instead of relying on [auto-generation.](basics/defining_environment.md#automatic-dockerfile-generation) -#### `--cuda` +#### `--base-image`, `-b` -Make cuda drivers available to task code. - -#### `--opencl` - -Make opencl drivers available to task code. +One of `opencl`, `cuda`, `docker`. If provided, the workflow will be built using with the specified software in the environment. These options are mutually exclusive. ## `latch register` diff --git a/latch_cli/main.py b/latch_cli/main.py index 40a5142f..214d2700 100644 --- a/latch_cli/main.py +++ b/latch_cli/main.py @@ -162,6 +162,7 @@ def login(connection: Optional[str]): ) @click.option( "--base-image", + "-b", help="Which base image to use for the Dockerfile.", type=click.Choice( list(BaseImageOptions._member_names_), From 63497299cc3dc652d8a9e23244fe28d0eaac12b9 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 15:21:56 -0800 Subject: [PATCH 05/19] fix: doc updates --- docs/source/basics/defining_environment.md | 2 +- docs/source/basics/environment/docker_recipes.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index 2e96454b..568e8cd8 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -260,7 +260,7 @@ The default `.dockerignore` includes files auto-generated by Latch. ## Docker Limitations -Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `mount` and `chroot` among others. +Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `chown` and `chroot` among others. However, we use a [container runtime](https://github.com/nestybox/sysbox) that allows system software that does not require these kernel capabilities to run. For example, users can run docker in docker. Note: `gpu` and `large-gpu` tasks do not support running system software. --- diff --git a/docs/source/basics/environment/docker_recipes.md b/docs/source/basics/environment/docker_recipes.md index 2dbf615e..3941a515 100644 --- a/docs/source/basics/environment/docker_recipes.md +++ b/docs/source/basics/environment/docker_recipes.md @@ -9,3 +9,7 @@ RUN curl -L https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.4.4/bowt unzip bowtie2-2.4.4.zip &&\ mv bowtie2-2.4.4-linux-x86_64 bowtie2 ``` + +## Run Docker in Docker + +Latch provides a base image which includes Docker. Specify the `--base-image docker` argument to `latch init` to use the image. Latch provides a comprehensive example of running a containerized `bowtie2` aligner in a Latch workflow, which can be found by running `latch init --template docker ...`. From 22865f5ea89403b3b641c0aa8b0ae066b1bb8723 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 15:44:10 -0800 Subject: [PATCH 06/19] fix: small things --- docs/source/api/latch_cli.rst | 16 +++++++-------- latch/types/directory.py | 1 - latch_cli/docker_utils/__init__.py | 12 +++++++---- latch_cli/services/init/init.py | 16 +++------------ latch_cli/types.py | 13 ------------ latch_cli/workflow_config.py | 33 ++++++++++++++++++++++++++++++ setup.py | 2 +- 7 files changed, 53 insertions(+), 40 deletions(-) delete mode 100644 latch_cli/types.py create mode 100644 latch_cli/workflow_config.py diff --git a/docs/source/api/latch_cli.rst b/docs/source/api/latch_cli.rst index adefcdc4..38650c53 100644 --- a/docs/source/api/latch_cli.rst +++ b/docs/source/api/latch_cli.rst @@ -25,6 +25,14 @@ latch\_cli.click\_utils module :undoc-members: :show-inheritance: +latch\_cli.config module +------------------------ + +.. automodule:: latch_cli.config + :members: + :undoc-members: + :show-inheritance: + latch\_cli.constants module --------------------------- @@ -57,14 +65,6 @@ latch\_cli.tui module :undoc-members: :show-inheritance: -latch\_cli.types module ------------------------ - -.. automodule:: latch_cli.types - :members: - :undoc-members: - :show-inheritance: - latch\_cli.utils module ----------------------- diff --git a/latch/types/directory.py b/latch/types/directory.py index ce2eee6c..30449701 100644 --- a/latch/types/directory.py +++ b/latch/types/directory.py @@ -148,7 +148,6 @@ def to_python_value( "Casting from Pathlike to LatchDir is currently not supported." ) - while get_origin(expected_python_type) == Annotated: expected_python_type = get_args(expected_python_type)[0] diff --git a/latch_cli/docker_utils/__init__.py b/latch_cli/docker_utils/__init__.py index eb8a506d..2edb147d 100644 --- a/latch_cli/docker_utils/__init__.py +++ b/latch_cli/docker_utils/__init__.py @@ -6,10 +6,11 @@ from textwrap import dedent from typing import List +import click import yaml from latch_cli.constants import latch_constants -from latch_cli.types import LatchWorkflowConfig +from latch_cli.workflow_config import LatchWorkflowConfig, create_and_write_config class DockerCmdBlockOrder(str, Enum): @@ -202,9 +203,12 @@ def generate_dockerfile(pkg_root: Path, outfile: Path) -> None: print(" - base image:", config.base_image) print(" - latch version:", config.latch_version) except FileNotFoundError as e: - raise RuntimeError( - "Could not find a .latch/config file in the supplied directory. If your workflow was created prior to release 2.13.0, you may need to run `latch init` to generate a .latch/config file." - ) from e + print( + "Could not find a .latch/config file in the supplied directory. Recreating configuration." + ) + create_and_write_config(latch_constants.base_image, pkg_root) + with open(pkg_root / latch_constants.pkg_config) as f: + config: LatchWorkflowConfig = LatchWorkflowConfig(**json.load(f)) with outfile.open("w") as f: f.write("\n".join(get_prologue(config)) + "\n\n") diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index 0ac8c5f8..d06e6e81 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -4,7 +4,6 @@ import re import shutil import subprocess -from dataclasses import asdict from datetime import datetime from enum import Enum from pathlib import Path @@ -16,7 +15,7 @@ from latch_cli.constants import latch_constants from latch_cli.docker_utils import generate_dockerfile from latch_cli.tui import select_tui -from latch_cli.types import LatchWorkflowConfig +from latch_cli.workflow_config import create_and_write_config def _get_boilerplate(pkg_root: Path, source_path: Path): @@ -304,19 +303,10 @@ def init( if template == "docker": base_image = base_image.replace("latch-base", "latch-base-docker") - config = LatchWorkflowConfig( - latch_version=get_distribution("latch").version, - base_image=base_image, - date=datetime.now().isoformat(), - ) - - (pkg_root / ".latch").mkdir(exist_ok=True) - - with open(pkg_root / latch_constants.pkg_config, "w") as f: - f.write(json.dumps(asdict(config))) - template_func(pkg_root) + create_and_write_config(base_image, pkg_root) + if expose_dockerfile: generate_dockerfile(pkg_root, pkg_root / "Dockerfile") diff --git a/latch_cli/types.py b/latch_cli/types.py deleted file mode 100644 index 7a44d7dc..00000000 --- a/latch_cli/types.py +++ /dev/null @@ -1,13 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class LatchWorkflowConfig: - """Configuration for a Latch workflow.""" - - latch_version: str - """Version of the Latch SDK used to initialize the workflow""" - base_image: str - """Exact version of the included workflow base image""" - date: str - """Timestamp of the `latch init` call""" diff --git a/latch_cli/workflow_config.py b/latch_cli/workflow_config.py new file mode 100644 index 00000000..cd4012b2 --- /dev/null +++ b/latch_cli/workflow_config.py @@ -0,0 +1,33 @@ +import json +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path + +from pkg_resources import get_distribution + +from latch_cli.constants import latch_constants + + +@dataclass(frozen=True) +class LatchWorkflowConfig: + """Configuration for a Latch workflow.""" + + latch_version: str + """Version of the Latch SDK used to initialize the workflow""" + base_image: str + """Exact version of the included workflow base image""" + date: str + """Timestamp of the `latch init` call""" + + +def create_and_write_config(base_image: str, pkg_root: Path): + config = LatchWorkflowConfig( + latch_version=get_distribution("latch").version, + base_image=base_image, + date=datetime.now().isoformat(), + ) + + (pkg_root / ".latch").mkdir(exist_ok=True) + + with open(pkg_root / latch_constants.pkg_config, "w") as f: + f.write(json.dumps(asdict(config))) diff --git a/setup.py b/setup.py index 99bbd93f..a7892daf 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name="latch", - version="v2.13.1", + version="v2.13.2", author_email="kenny@latch.bio", description="The Latchbio SDK", packages=find_packages(), From 046fd68493c4fc1dbbd5920bf15cecf2fe602bf0 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 16:50:52 -0800 Subject: [PATCH 07/19] fix: small things --- docs/source/api/latch_cli.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/api/latch_cli.rst b/docs/source/api/latch_cli.rst index 38650c53..89e56dee 100644 --- a/docs/source/api/latch_cli.rst +++ b/docs/source/api/latch_cli.rst @@ -25,14 +25,6 @@ latch\_cli.click\_utils module :undoc-members: :show-inheritance: -latch\_cli.config module ------------------------- - -.. automodule:: latch_cli.config - :members: - :undoc-members: - :show-inheritance: - latch\_cli.constants module --------------------------- @@ -73,6 +65,14 @@ latch\_cli.utils module :undoc-members: :show-inheritance: +latch\_cli.workflow\_config module +---------------------------------- + +.. automodule:: latch_cli.workflow_config + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- From 454f6d989aaafc1ea21582e1e616415e599ac04b Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Mon, 20 Feb 2023 16:57:47 -0800 Subject: [PATCH 08/19] fix: changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6bdbf4..b5760fa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,18 @@ Types of changes ### Fixed +* Create latch config file if it does not exist at registration or develop time + +### Added + +* `latch init`: Docker in Docker template workflow +* `latch init`: Docker base image +* Small, medium, and large tasks run with sysbox runtime allowing the execution of system code + +## 2.13.1 - 2023-02-17 + +### Fixed + * Add latch/latch_cli/services/init/common to pypi release ## 2.13.0 - 2023-02-17 From 7e767980f36d7e726d034a7829189f75e3ac5b41 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 10:15:26 -0800 Subject: [PATCH 09/19] fix: Ayush PR comments --- CHANGELOG.md | 2 +- docs/source/basics/defining_environment.md | 4 +-- docs/source/subcommands.md | 2 +- latch_cli/docker_utils/__init__.py | 4 +-- latch_cli/main.py | 1 + latch_cli/services/init/init.py | 32 +++++++--------------- latch_cli/workflow_config.py | 16 +++++++++-- 7 files changed, 31 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5760fa9..e0f9786b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ Types of changes * `latch init`: Docker in Docker template workflow * `latch init`: Docker base image -* Small, medium, and large tasks run with sysbox runtime allowing the execution of system code +* Small, medium, and large tasks run with sysbox runtime allowing the execution system-level software such as systemd, Docker, Kubernetes, K3s, buildx, legacy apps, and more. ## 2.13.1 - 2023-02-17 diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index 568e8cd8..e6635c21 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -4,7 +4,7 @@ Workflow code is rarely free of dependencies. It may require python or system pa Latch manages the execution environment of a workflow using Dockerfiles. A Dockerfile specifies the environment in which the workflow runs. -Latch has four [base images](https://docs.docker.com/build/building/base-images/), one bare minimum, one with CUDA drivers, one with OPENCL drivers, and one with the Docker software pre-installed. To use the CUDA, OPENCL, or Docker base image, use the `--base-image` flag when running `latch init`. +Latch has four [base images](https://docs.docker.com/build/building/base-images/), one bare minimum, one with CUDA drivers, one with OPENCL drivers, and one with Docker pre-installed. To use the CUDA, OPENCL, or Docker base image, provide the specific value using the `--base-image` option when running `latch init`. The workflow environment is encapsulated in [a Docker container](https://en.wikipedia.org/wiki/Docker_(software)), which is created from a recipe defined in [a text document called Dockerfile.](https://docs.docker.com/engine/reference/builder/) In most cases these recipes are repetitive and unnecessarily complicated, so Latch will automatically generate one using conventional dependency lists and heuristics. To use a handwritten Dockerfile, [run the eject command](#ejecting-auto-generation). @@ -260,7 +260,7 @@ The default `.dockerignore` includes files auto-generated by Latch. ## Docker Limitations -Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `chown` and `chroot` among others. However, we use a [container runtime](https://github.com/nestybox/sysbox) that allows system software that does not require these kernel capabilities to run. For example, users can run docker in docker. Note: `gpu` and `large-gpu` tasks do not support running system software. +Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `chown` and `chroot` among others. However, for non-GPU tasks, we use a [container runtime](https://github.com/nestybox/sysbox) that gets around this issue. For example, a user can run a Docker container inside a `small_task` (Docker in Docker), which is disallowed using the normal Docker runtime. --- diff --git a/docs/source/subcommands.md b/docs/source/subcommands.md index 02df81d8..f4d6d623 100644 --- a/docs/source/subcommands.md +++ b/docs/source/subcommands.md @@ -22,7 +22,7 @@ Generate a Dockerfile for the workflow instead of relying on [auto-generation.]( #### `--base-image`, `-b` -One of `opencl`, `cuda`, `docker`. If provided, the workflow will be built using with the specified software in the environment. These options are mutually exclusive. +One of `opencl`, `cuda`, `docker`, `default`. If provided, the workflow will be built using with the specified software in the environment. These options are mutually exclusive. If not provided or `default`, will use a bare-minimum image. ## `latch register` diff --git a/latch_cli/docker_utils/__init__.py b/latch_cli/docker_utils/__init__.py index 2edb147d..d234f9a8 100644 --- a/latch_cli/docker_utils/__init__.py +++ b/latch_cli/docker_utils/__init__.py @@ -204,9 +204,9 @@ def generate_dockerfile(pkg_root: Path, outfile: Path) -> None: print(" - latch version:", config.latch_version) except FileNotFoundError as e: print( - "Could not find a .latch/config file in the supplied directory. Recreating configuration." + "Could not find a .latch/config file in the supplied directory. Creating configuration." ) - create_and_write_config(latch_constants.base_image, pkg_root) + create_and_write_config(pkg_root) with open(pkg_root / latch_constants.pkg_config) as f: config: LatchWorkflowConfig = LatchWorkflowConfig(**json.load(f)) diff --git a/latch_cli/main.py b/latch_cli/main.py index 214d2700..81466f9a 100644 --- a/latch_cli/main.py +++ b/latch_cli/main.py @@ -168,6 +168,7 @@ def login(connection: Optional[str]): list(BaseImageOptions._member_names_), case_sensitive=False, ), + default="default", ) def init( pkg_name: str, diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index d06e6e81..8142f4fe 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -1,16 +1,13 @@ """Service to initialize boilerplate.""" -import json import re import shutil import subprocess -from datetime import datetime from enum import Enum from pathlib import Path from typing import Callable, Optional import click -from pkg_resources import get_distribution from latch_cli.constants import latch_constants from latch_cli.docker_utils import generate_dockerfile @@ -83,10 +80,6 @@ def _get_example_reference(pkg_root: Path): def _gen_assemble_and_sort(pkg_root: Path): - import boto3 - from botocore import UNSIGNED - from botocore.config import Config - pkg_root = pkg_root.resolve() source_path = Path(__file__).parent / "assemble_and_sort" @@ -152,10 +145,6 @@ def _gen_example_conda(pkg_root: Path): def _gen_example_docker(pkg_root: Path): - import boto3 - from botocore import UNSIGNED - from botocore.config import Config - pkg_root = pkg_root.resolve() source_path = Path(__file__).parent / "assemble_and_sort" @@ -167,8 +156,6 @@ def _gen_example_docker(pkg_root: Path): (pkg_root / ".env").unlink() - print() - option_map = { "Empty workflow": _gen_template, @@ -189,6 +176,7 @@ def _gen_example_docker(pkg_root: Path): class BaseImageOptions(Enum): + default = "default" cuda = "cuda" opencl = "opencl" docker = "docker" @@ -198,7 +186,7 @@ def init( pkg_name: str, template: Optional[str], expose_dockerfile: bool = True, - base_image_type: Optional[str] = None, + base_image_type_str: str = "default", ) -> bool: """Creates boilerplate workflow files in the user's working directory. @@ -278,6 +266,13 @@ def init( else template_flag_to_option[template] ) + if base_image_type_str not in BaseImageOptions.__members__: + raise ValueError( + f"Invalid base image type: {base_image_type_str}. Must be one of {list(BaseImageOptions.__members__.keys())}" + ) + + base_image_type = BaseImageOptions.__members__[base_image_type_str] + if selected_option is None: return False @@ -296,16 +291,9 @@ def init( ): return False - base_image = latch_constants.base_image - if base_image_type is not None: - base_image = base_image.replace("latch-base", f"latch-base-{base_image_type}") - - if template == "docker": - base_image = base_image.replace("latch-base", "latch-base-docker") - template_func(pkg_root) - create_and_write_config(base_image, pkg_root) + create_and_write_config(pkg_root, base_image_type) if expose_dockerfile: generate_dockerfile(pkg_root, pkg_root / "Dockerfile") diff --git a/latch_cli/workflow_config.py b/latch_cli/workflow_config.py index cd4012b2..6e7a772c 100644 --- a/latch_cli/workflow_config.py +++ b/latch_cli/workflow_config.py @@ -2,10 +2,12 @@ from dataclasses import asdict, dataclass from datetime import datetime from pathlib import Path +from typing import Optional from pkg_resources import get_distribution from latch_cli.constants import latch_constants +from latch_cli.services.init.init import BaseImageOptions @dataclass(frozen=True) @@ -20,10 +22,20 @@ class LatchWorkflowConfig: """Timestamp of the `latch init` call""" -def create_and_write_config(base_image: str, pkg_root: Path): +def create_and_write_config( + pkg_root: Path, base_image_type: BaseImageOptions = BaseImageOptions.default +): + + base_image = latch_constants.base_image + + if base_image_type != BaseImageOptions.default: + base_image = base_image.replace( + "latch-base", f"latch-base-{base_image_type.name}" + ) + config = LatchWorkflowConfig( latch_version=get_distribution("latch").version, - base_image=base_image, + base_image=base_image if base_image is not None else latch_constants.base_image, date=datetime.now().isoformat(), ) From 3af0bda19108ae08e5ee459785deba535447c6f7 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 10:26:20 -0800 Subject: [PATCH 10/19] fix: fix docstring --- latch_cli/services/init/init.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index 8142f4fe..ec21c594 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -205,6 +205,12 @@ def init( time and the user will not be able to modify it. At any point, the user can switch modes by executing `latch dockerfile .` in the workflow directory. + base_image_type_str: The base image to use for the workflow. Default value + is "default". The following options are available: + * "default": Bare-minimum image to execute a Latch workflow + * "cuda": adds CUDA drivers + * "opencl": adds OpenCL drivers + * "docker": adds Docker daemon Example: >>> init("test-workflow", "empty", False) From 70d10c6c0d2e7eef4b286930189eeec8aaf2f5df Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 10:26:48 -0800 Subject: [PATCH 11/19] Update CHANGELOG.md Co-authored-by: Ayush Kamat <34531970+ayushkamat@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0f9786b..78ce36e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ Types of changes * `latch init`: Docker in Docker template workflow * `latch init`: Docker base image -* Small, medium, and large tasks run with sysbox runtime allowing the execution system-level software such as systemd, Docker, Kubernetes, K3s, buildx, legacy apps, and more. +* Small, medium, and large tasks run with sysbox runtime allowing the execution of system-level software such as `systemd`, Docker, Kubernetes, K3s, `buildx`, legacy apps, and more. ## 2.13.1 - 2023-02-17 From a0a15da78b165f15d7b761e40dcd917b99673b77 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 10:27:11 -0800 Subject: [PATCH 12/19] Update docs/source/basics/defining_environment.md Co-authored-by: Ayush Kamat <34531970+ayushkamat@users.noreply.github.com> --- docs/source/basics/defining_environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index e6635c21..d6f94309 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -260,7 +260,7 @@ The default `.dockerignore` includes files auto-generated by Latch. ## Docker Limitations -Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `chown` and `chroot` among others. However, for non-GPU tasks, we use a [container runtime](https://github.com/nestybox/sysbox) that gets around this issue. For example, a user can run a Docker container inside a `small_task` (Docker in Docker), which is disallowed using the normal Docker runtime. +Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `chown` and `chroot` among others. However, for non-GPU tasks, we use a [container runtime](https://github.com/nestybox/sysbox) that gets around this issue. For example, a user can run a Docker container inside a `small_task` (Docker in Docker), which is disallowed when using the normal Docker runtime. --- From 2fe4acfec967e0258aff6fedfca184b37883ede0 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 10:58:52 -0800 Subject: [PATCH 13/19] fix: Max's comments --- CHANGELOG.md | 4 ++-- docs/source/basics/defining_environment.md | 10 +++------- docs/source/basics/environment/docker_recipes.md | 2 ++ docs/source/subcommands.md | 8 +++++++- latch_cli/docker_utils/__init__.py | 6 +++--- latch_cli/services/init/example_docker/assemble.py | 2 -- latch_cli/services/init/init.py | 12 ++++++------ latch_cli/workflow_config.py | 2 +- 8 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78ce36e8..463fa3c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,13 +18,13 @@ Types of changes ### Fixed -* Create latch config file if it does not exist at registration or develop time +* Internal state file should be automatically created when running `latch register` and `latch develop` ### Added * `latch init`: Docker in Docker template workflow * `latch init`: Docker base image -* Small, medium, and large tasks run with sysbox runtime allowing the execution of system-level software such as `systemd`, Docker, Kubernetes, K3s, `buildx`, legacy apps, and more. +* Small, medium, and large tasks use a [Sysbox runtime](https://github.com/nestybox/sysbox) to run Docker and other system software within task containers ## 2.13.1 - 2023-02-17 diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index d6f94309..b6a754c2 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -2,11 +2,7 @@ Workflow code is rarely free of dependencies. It may require python or system packages or make use of environment variables. For example, a task that downloads compressed reference data from AWS S3 will need the `aws-cli` and `unzip` [APT](https://en.wikipedia.org/wiki/APT_(software)) packages, then use the `pyyaml` python package to read the included metadata. -Latch manages the execution environment of a workflow using Dockerfiles. A Dockerfile specifies the environment in which the workflow runs. - -Latch has four [base images](https://docs.docker.com/build/building/base-images/), one bare minimum, one with CUDA drivers, one with OPENCL drivers, and one with Docker pre-installed. To use the CUDA, OPENCL, or Docker base image, provide the specific value using the `--base-image` option when running `latch init`. - -The workflow environment is encapsulated in [a Docker container](https://en.wikipedia.org/wiki/Docker_(software)), which is created from a recipe defined in [a text document called Dockerfile.](https://docs.docker.com/engine/reference/builder/) In most cases these recipes are repetitive and unnecessarily complicated, so Latch will automatically generate one using conventional dependency lists and heuristics. To use a handwritten Dockerfile, [run the eject command](#ejecting-auto-generation). +The workflow environment is encapsulated in [a Docker container](https://en.wikipedia.org/wiki/Docker_(software)), which is created from a recipe defined in [a text document called Dockerfile.](https://docs.docker.com/engine/reference/builder/). Latch provides [four baseline environments](../subcommands.md#base-image--b) which each latch workflows inherits from. In most cases, modifying the `Dockefile` manually is unecessary, so Latch will automatically generate one using conventional dependency lists and heuristics. To use a handwritten Dockerfile, [run the eject command](#ejecting-auto-generation). ## Automatic Dockerfile Generation @@ -258,9 +254,9 @@ To exclude files from the build use [a `.dockerignore`.](https://docs.docker.com The default `.dockerignore` includes files auto-generated by Latch. -## Docker Limitations +## GPU Task Limitations -Docker containers have limitations on some system administration functions. While each workflow runs as a genuine `root` user (UID 0), commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied". This includes `chown` and `chroot` among others. However, for non-GPU tasks, we use a [container runtime](https://github.com/nestybox/sysbox) that gets around this issue. For example, a user can run a Docker container inside a `small_task` (Docker in Docker), which is disallowed when using the normal Docker runtime. +Commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied" in GPU tasks (`small-gpu-task`, `large-gpu-task`). This includes `chown` and `chroot` among others. --- diff --git a/docs/source/basics/environment/docker_recipes.md b/docs/source/basics/environment/docker_recipes.md index 3941a515..0cadd056 100644 --- a/docs/source/basics/environment/docker_recipes.md +++ b/docs/source/basics/environment/docker_recipes.md @@ -13,3 +13,5 @@ RUN curl -L https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.4.4/bowt ## Run Docker in Docker Latch provides a base image which includes Docker. Specify the `--base-image docker` argument to `latch init` to use the image. Latch provides a comprehensive example of running a containerized `bowtie2` aligner in a Latch workflow, which can be found by running `latch init --template docker ...`. + +Use [`--base-image docker` with `latch init`](../../subcommands.md##latch-init) to use a base workflow environment which includes Docker. An example of running a containerized `bowtie2` aligner in a Latch workflow can be found using `latch init --template docker my_bowtie2_example`. diff --git a/docs/source/subcommands.md b/docs/source/subcommands.md index f4d6d623..ee3fd6e5 100644 --- a/docs/source/subcommands.md +++ b/docs/source/subcommands.md @@ -22,7 +22,13 @@ Generate a Dockerfile for the workflow instead of relying on [auto-generation.]( #### `--base-image`, `-b` -One of `opencl`, `cuda`, `docker`, `default`. If provided, the workflow will be built using with the specified software in the environment. These options are mutually exclusive. If not provided or `default`, will use a bare-minimum image. +Each environment is build on one of the following base distributions: +- `default` with no additional dependencies +- `cuda` with Nvidia CUDA/cuDNN (cuda 11.4.2, cudnn 8) drivers +- `opencl` with OpenCL (ubuntu 18.04) drivers +- `docker` with the Docker daemon + +Only one option can be given at a time. If not provided or `default`, only the bare minimum packages to execute the workflow will be installed. ## `latch register` diff --git a/latch_cli/docker_utils/__init__.py b/latch_cli/docker_utils/__init__.py index d234f9a8..8c59e5ef 100644 --- a/latch_cli/docker_utils/__init__.py +++ b/latch_cli/docker_utils/__init__.py @@ -198,16 +198,16 @@ def generate_dockerfile(pkg_root: Path, outfile: Path) -> None: print("Generating Dockerfile") try: - with open(pkg_root / latch_constants.pkg_config) as f: + with (pkg_root / latch_constants.pkg_config).open("r") as f: config: LatchWorkflowConfig = LatchWorkflowConfig(**json.load(f)) print(" - base image:", config.base_image) print(" - latch version:", config.latch_version) except FileNotFoundError as e: print( - "Could not find a .latch/config file in the supplied directory. Creating configuration." + "Could not find a .latch/config file in the supplied directory. Creating configuration" ) create_and_write_config(pkg_root) - with open(pkg_root / latch_constants.pkg_config) as f: + with (pkg_root / latch_constants.pkg_config).open("r") as f: config: LatchWorkflowConfig = LatchWorkflowConfig(**json.load(f)) with outfile.open("w") as f: diff --git a/latch_cli/services/init/example_docker/assemble.py b/latch_cli/services/init/example_docker/assemble.py index cea2fa71..fde13542 100644 --- a/latch_cli/services/init/example_docker/assemble.py +++ b/latch_cli/services/init/example_docker/assemble.py @@ -44,13 +44,11 @@ def assembly_task( ] try: - # We use shell=True for all the benefits of pipes and other shell features. # When using shell=True, we pass the entire command as a single string as # opposed to a list since the shell will parse the string into a list # using its own rules. subprocess.run(" ".join(bowtie2_cmd), shell=True, check=True) except subprocess.CalledProcessError as e: - # will display in the messages tab of the execution graph for the assembly_task node message( "error", {"title": "Bowtie2 Failed", "body": f"Error: {str(e)}"}, diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index ec21c594..9f38ea94 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -175,7 +175,7 @@ def _gen_example_docker(pkg_root: Path): } -class BaseImageOptions(Enum): +class BaseImageOptions(str, Enum): default = "default" cuda = "cuda" opencl = "opencl" @@ -205,12 +205,12 @@ def init( time and the user will not be able to modify it. At any point, the user can switch modes by executing `latch dockerfile .` in the workflow directory. - base_image_type_str: The base image to use for the workflow. Default value + base_image_type_str: Base image to use for the workflow. Default value is "default". The following options are available: - * "default": Bare-minimum image to execute a Latch workflow - * "cuda": adds CUDA drivers - * "opencl": adds OpenCL drivers - * "docker": adds Docker daemon + * "default": with no additional dependencies + * "cuda": with Nvidia CUDA/cuDNN (cuda 11.4.2, cudnn 8) drivers + * "opencl": with OpenCL (ubuntu 18.04) drivers + * "docker": with the Docker daemon Example: >>> init("test-workflow", "empty", False) diff --git a/latch_cli/workflow_config.py b/latch_cli/workflow_config.py index 6e7a772c..e88b0be3 100644 --- a/latch_cli/workflow_config.py +++ b/latch_cli/workflow_config.py @@ -41,5 +41,5 @@ def create_and_write_config( (pkg_root / ".latch").mkdir(exist_ok=True) - with open(pkg_root / latch_constants.pkg_config, "w") as f: + with (pkg_root / latch_constants.pkg_config).open("w") as f: f.write(json.dumps(asdict(config))) From 4f13e17d2cbd970ff6cd045663c2a7ebe3646d84 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 11:05:13 -0800 Subject: [PATCH 14/19] fix: resolve circular import --- latch_cli/main.py | 3 ++- latch_cli/services/init/init.py | 11 +---------- latch_cli/workflow_config.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/latch_cli/main.py b/latch_cli/main.py index 81466f9a..be31048a 100644 --- a/latch_cli/main.py +++ b/latch_cli/main.py @@ -10,8 +10,9 @@ import latch_cli.click_utils from latch_cli.exceptions.handler import CrashHandler -from latch_cli.services.init.init import BaseImageOptions, template_flag_to_option +from latch_cli.services.init.init import template_flag_to_option from latch_cli.utils import get_latest_package_version, get_local_package_version +from latch_cli.workflow_config import BaseImageOptions latch_cli.click_utils.patch() diff --git a/latch_cli/services/init/init.py b/latch_cli/services/init/init.py index 9f38ea94..71321ae5 100644 --- a/latch_cli/services/init/init.py +++ b/latch_cli/services/init/init.py @@ -3,16 +3,14 @@ import re import shutil import subprocess -from enum import Enum from pathlib import Path from typing import Callable, Optional import click -from latch_cli.constants import latch_constants from latch_cli.docker_utils import generate_dockerfile from latch_cli.tui import select_tui -from latch_cli.workflow_config import create_and_write_config +from latch_cli.workflow_config import BaseImageOptions, create_and_write_config def _get_boilerplate(pkg_root: Path, source_path: Path): @@ -175,13 +173,6 @@ def _gen_example_docker(pkg_root: Path): } -class BaseImageOptions(str, Enum): - default = "default" - cuda = "cuda" - opencl = "opencl" - docker = "docker" - - def init( pkg_name: str, template: Optional[str], diff --git a/latch_cli/workflow_config.py b/latch_cli/workflow_config.py index e88b0be3..c805864e 100644 --- a/latch_cli/workflow_config.py +++ b/latch_cli/workflow_config.py @@ -1,13 +1,19 @@ import json from dataclasses import asdict, dataclass from datetime import datetime +from enum import Enum from pathlib import Path -from typing import Optional from pkg_resources import get_distribution from latch_cli.constants import latch_constants -from latch_cli.services.init.init import BaseImageOptions + + +class BaseImageOptions(str, Enum): + default = "default" + cuda = "cuda" + opencl = "opencl" + docker = "docker" @dataclass(frozen=True) From 4298db3b6933021cc68fe14de44497ec927a99f2 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 11:14:47 -0800 Subject: [PATCH 15/19] Update CHANGELOG.md Co-authored-by: Ayush Kamat <34531970+ayushkamat@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 463fa3c1..062d53d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ Types of changes * `latch init`: Docker in Docker template workflow * `latch init`: Docker base image -* Small, medium, and large tasks use a [Sysbox runtime](https://github.com/nestybox/sysbox) to run Docker and other system software within task containers +* Small, medium, and large tasks use the [Sysbox runtime](https://github.com/nestybox/sysbox) to run Docker and other system software within task containers ## 2.13.1 - 2023-02-17 From abd2c48b1a4a88a0795aecece7bb6b78a6298beb Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 11:15:16 -0800 Subject: [PATCH 16/19] Update docs/source/basics/defining_environment.md Co-authored-by: Ayush Kamat <34531970+ayushkamat@users.noreply.github.com> --- docs/source/basics/defining_environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index b6a754c2..fe4a8c99 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -2,7 +2,7 @@ Workflow code is rarely free of dependencies. It may require python or system packages or make use of environment variables. For example, a task that downloads compressed reference data from AWS S3 will need the `aws-cli` and `unzip` [APT](https://en.wikipedia.org/wiki/APT_(software)) packages, then use the `pyyaml` python package to read the included metadata. -The workflow environment is encapsulated in [a Docker container](https://en.wikipedia.org/wiki/Docker_(software)), which is created from a recipe defined in [a text document called Dockerfile.](https://docs.docker.com/engine/reference/builder/). Latch provides [four baseline environments](../subcommands.md#base-image--b) which each latch workflows inherits from. In most cases, modifying the `Dockefile` manually is unecessary, so Latch will automatically generate one using conventional dependency lists and heuristics. To use a handwritten Dockerfile, [run the eject command](#ejecting-auto-generation). +The workflow environment is encapsulated in [a Docker container](https://en.wikipedia.org/wiki/Docker_(software)), which is created from a recipe defined in [a text document named Dockerfile.](https://docs.docker.com/engine/reference/builder/). Latch provides [four baseline environments](../subcommands.md#base-image--b) which each latch workflow inherits from. In most cases, modifying the `Dockefile` manually is unnecessary, so Latch will automatically generate one using conventional dependency lists and heuristics. To use a handwritten Dockerfile, [run the eject command](#ejecting-auto-generation). ## Automatic Dockerfile Generation From 7a2c6139bcb3992d9c4e03c195fae9d5d532393a Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 11:15:27 -0800 Subject: [PATCH 17/19] Update docs/source/basics/environment/docker_recipes.md Co-authored-by: Ayush Kamat <34531970+ayushkamat@users.noreply.github.com> --- docs/source/basics/environment/docker_recipes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basics/environment/docker_recipes.md b/docs/source/basics/environment/docker_recipes.md index 0cadd056..f33c08db 100644 --- a/docs/source/basics/environment/docker_recipes.md +++ b/docs/source/basics/environment/docker_recipes.md @@ -12,6 +12,6 @@ RUN curl -L https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.4.4/bowt ## Run Docker in Docker -Latch provides a base image which includes Docker. Specify the `--base-image docker` argument to `latch init` to use the image. Latch provides a comprehensive example of running a containerized `bowtie2` aligner in a Latch workflow, which can be found by running `latch init --template docker ...`. +Latch provides a base image which includes Docker. Specify the `--base-image docker` argument to `latch init` to use the image. Latch also provides a comprehensive example of running a containerized `bowtie2` aligner in a Latch workflow, which can be found by running `latch init --template docker ...`. Use [`--base-image docker` with `latch init`](../../subcommands.md##latch-init) to use a base workflow environment which includes Docker. An example of running a containerized `bowtie2` aligner in a Latch workflow can be found using `latch init --template docker my_bowtie2_example`. From f7f49e0e0273a8b57a0363cb74c41934ed029d86 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 11:17:21 -0800 Subject: [PATCH 18/19] fix: silly typos --- docs/source/basics/environment/docker_recipes.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/basics/environment/docker_recipes.md b/docs/source/basics/environment/docker_recipes.md index 0cadd056..66dce917 100644 --- a/docs/source/basics/environment/docker_recipes.md +++ b/docs/source/basics/environment/docker_recipes.md @@ -12,6 +12,4 @@ RUN curl -L https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.4.4/bowt ## Run Docker in Docker -Latch provides a base image which includes Docker. Specify the `--base-image docker` argument to `latch init` to use the image. Latch provides a comprehensive example of running a containerized `bowtie2` aligner in a Latch workflow, which can be found by running `latch init --template docker ...`. - Use [`--base-image docker` with `latch init`](../../subcommands.md##latch-init) to use a base workflow environment which includes Docker. An example of running a containerized `bowtie2` aligner in a Latch workflow can be found using `latch init --template docker my_bowtie2_example`. From 2564b1cf6e10dfe619f41a08f0a69860d5772c14 Mon Sep 17 00:00:00 2001 From: AidanAbd Date: Tue, 21 Feb 2023 11:20:49 -0800 Subject: [PATCH 19/19] fix: small --- docs/source/basics/defining_environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/basics/defining_environment.md b/docs/source/basics/defining_environment.md index fe4a8c99..01e476cd 100644 --- a/docs/source/basics/defining_environment.md +++ b/docs/source/basics/defining_environment.md @@ -256,7 +256,7 @@ The default `.dockerignore` includes files auto-generated by Latch. ## GPU Task Limitations -Commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied" in GPU tasks (`small-gpu-task`, `large-gpu-task`). This includes `chown` and `chroot` among others. +Commands that require certain [kernel capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) will fail with "Permission denied" in GPU tasks (`small-gpu-task`, `large-gpu-task`). This includes `mount` and `chroot` among others. ---