macOS compatibility improvements (#329)

*- default `--runtime-cpu-max` and `--runtime-memory-max` to the resources available in Docker for Mac's virtual machine, rather than the entire host
* `miniwdl run_self_test` failure displays hint on overriding `TMPDIR` (#324)
* logging & documentation refinements

Some problems remain, particularly with the full test suite; tracking in #145

CONTRIBUTING.md

@@ -2,16 +2,19 @@
 
 Feedback and contributions to miniwdl are welcome, via issues and pull requests on this repository.
 
+* [Online documentation](https://miniwdl.readthedocs.io/en/latest/) includes several "codelab" tutorials to start with
 * The [Project board](https://github.com/chanzuckerberg/miniwdl/projects/1) shows our current prioritization of [issues](https://github.com/chanzuckerberg/miniwdl/issues)
-* [Starter issues](https://github.com/chanzuckerberg/miniwdl/issues?q=is%3Aopen+is%3Aissue+label%3Astarter) are good potential entry points for new contributors
+* [Starter issues](https://github.com/chanzuckerberg/miniwdl/issues?q=is%3Aopen+is%3Aissue+label%3Astarter) are suitable entry points for new contributors
+* [Pull request template](https://github.com/chanzuckerberg/miniwdl/blob/master/.github/pull_request_template.md) includes a preparation checklist
 
-To set up your local development environment,
+To set up your Linux development environment,
 
-1. `git clone --recursive` this repository or your fork thereof
-2. Install dependencies as illustrated in the [Dockerfile](https://github.com/chanzuckerberg/miniwdl/blob/master/Dockerfile) (OS packages + PyPI packages listed in `requirements.txt` and `requirements.dev.txt`)
+1. `git clone --recursive` this repository or your fork thereof, and `cd` into it
+2. Install dependencies as illustrated in the [Dockerfile](https://github.com/chanzuckerberg/miniwdl/blob/master/Dockerfile) (OS packages + `pip3 install --user -r` both `requirements.txt` and `requirements.dev.txt`)
 3. Invoking user must have [permission to control Docker](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)
+4. Try `python3 -m WDL run_self_test` to test the configuration
 
-To invoke the `miniwdl` command-line interface from your working repository, e.g. `python3 -m WDL check ...` or `python3 -m WDL run ...`. Another possibility is to `pip3 install .` to install the `miniwdl` entry point with the current code revision (but leaving it installed!).
+Generally, `python3 -m WDL ...` invokes the equivalent of the `miniwdl ...` entry point for the local source tree. Another option is to `pip3 install .` to install the `miniwdl` entry point with the current code revision.
 
 The Makefile has a few typical flows:
 
@@ -22,7 +25,7 @@ The Makefile has a few typical flows:
 
 To quickly run only a relevant subset of the tests, you can e.g. `python3 -m unittest -f tests/test_5stdlib.py` or `python3 -m unittest -f tests.test_5stdlib.TestStdLib.test_glob`.
 
-Miniwdl's [online documentation](https://miniwdl.readthedocs.io/en/latest/) includes several codelab tutorials to start with. The [pull request template](https://github.com/chanzuckerberg/miniwdl/blob/master/.github/pull_request_template.md) includes a checklist for preparing your PR. Thank you!
+**macOS:** isn't preferred for miniwdl development due to some [test suite incompatibilities](https://github.com/chanzuckerberg/miniwdl/issues/145); but at least simple changes can be prototyped under macOS.
 
 ## Security
 

Makefile

@@ -1,5 +1,4 @@
 SHELL := /bin/bash
-PYTHON_PKG_BASE?=$(HOME)/.local
 export TMPDIR = /tmp
 
 test: check_check check unit_tests integration_tests
@@ -36,7 +35,7 @@ ci_unit_tests: unit_tests
 check:
 	pyre \
 		--search-path stubs \
-		--typeshed ${PYTHON_PKG_BASE}/lib/pyre_check/typeshed \
+		--typeshed `python3 -c 'import site; print(site.getuserbase())'`/lib/pyre_check/typeshed \
 		--show-parse-errors check
 	pylint -j `python3 -c 'import multiprocessing as mp; print(mp.cpu_count())'` --errors-only WDL
 

WDL/CLI.py

@@ -4,6 +4,7 @@
 # PYTHON_ARGCOMPLETE_OK
 import sys
 import os
+import platform
 import subprocess
 import tempfile
 import glob
@@ -26,8 +27,8 @@
     VERBOSE_LEVEL,
     NOTICE_LEVEL,
     install_coloredlogs,
-    ensure_swarm,
     parse_byte_size,
+    initialize_local_docker,
 )
 from ._util import StructuredLogMessage as _
 
@@ -438,12 +439,19 @@ def runner(
     logger = logging.getLogger("miniwdl-run")
     install_coloredlogs(logger)
 
+    versionlog = {}
     for pkg in ["miniwdl", "docker", "lark-parser", "argcomplete", "pygtail"]:
         try:
-            logger.debug(importlib_metadata.version(pkg))
+            versionlog[pkg] = str(importlib_metadata.version(pkg))
         except importlib_metadata.PackageNotFoundError:
-            logger.debug(f"{pkg} UNKNOWN")
-    logger.debug("dockerd: " + str(docker.from_env().version()))
+            versionlog[pkg] = "UNKNOWN"
+    logger.debug(_("package versions", **versionlog))
+
+    envlog = {}
+    for k in ["LANG", "SHELL", "USER", "HOME", "PWD", "TMPDIR"]:
+        if k in os.environ:
+            envlog[k] = os.environ[k]
+    logger.debug(_("environment", **envlog))
 
     rerun_sh = f"pushd {shellquote(os.getcwd())} && miniwdl {' '.join(shellquote(t) for t in sys.argv[1:])}; popd"
 
@@ -452,7 +460,7 @@ def runner(
         (k, kwargs[k])
         for k in ["copy_input_files", "run_dir", "runtime_cpu_max", "as_me", "max_tasks"]
     )
-    if runtime_memory_max:
+    if runtime_memory_max is not None:
         run_kwargs["runtime_memory_max"] = parse_byte_size(runtime_memory_max)
     if runtime_defaults:
         if runtime_defaults.lstrip()[0] == "{":
@@ -461,7 +469,18 @@ def runner(
             with open(runtime_defaults, "r") as infile:
                 run_kwargs["runtime_defaults"] = json.load(infile)
 
-    ensure_swarm(logger)
+    # initialize Docker
+    client = docker.from_env()
+    try:
+        logger.debug("dockerd :: " + json.dumps(client.version())[1:-1])
+        host_limits = initialize_local_docker(logger, client)
+    finally:
+        client.close()
+    if not isinstance(run_kwargs.get("runtime_cpu_max", None), int):
+        run_kwargs["runtime_cpu_max"] = host_limits["runtime_cpu_max"]
+    if not isinstance(run_kwargs.get("runtime_memory_max", None), int):
+        run_kwargs["runtime_memory_max"] = host_limits["runtime_memory_max"]
+    logger.debug(_("run_kwargs", **run_kwargs))
 
     # run & handle any errors
     try:
@@ -851,7 +870,12 @@ def run_self_test(**kwargs):
     except:
         atexit.register(
             lambda: print(
-                "* Hint: ensure Docker is installed, running, and user has permission to control it per https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user",
+                "* Hint: ensure Docker is installed & running"
+                + (
+                    ", and user has permission to control it per https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user"
+                    if platform.system() != "Darwin"
+                    else "; and on macOS override the environment variable TMPDIR=/tmp/"
+                ),
                 file=sys.stderr,
             )
         )
@@ -1177,6 +1201,13 @@ def scan(x):
 
     if uris:
         logging.basicConfig(level=NOTICE_LEVEL)
+        # initialize Docker
+        logger = logging.getLogger("miniwdl-localize")
+        client = docker.from_env()
+        try:
+            host_limits = initialize_local_docker(logger, client)
+        finally:
+            client.close()
 
         # cheesy trick: provide the list of URIs as File inputs to a dummy workflow, causing the
         # runtime to download them
@@ -1199,7 +1230,7 @@ def scan(x):
         subdir, outputs = runtime.run(
             localizer.workflow,
             values_from_json({"uris": list(uris)}, localizer.workflow.available_inputs),
-            **kwargs,
+            **host_limits,
         )
 
         # recover the mapping of URIs to downloaded files

WDL/_util.py

@@ -304,43 +304,6 @@ def poll() -> None:
         poll()
 
 
-@export
-def ensure_swarm(logger: logging.Logger) -> None:
-    client = docker.from_env()
-    try:
-        state = "(unknown)"
-        while True:
-            info = client.info()
-            if "Swarm" in info and "LocalNodeState" in info["Swarm"]:
-                state = info["Swarm"]["LocalNodeState"]
-
-            # https://github.com/moby/moby/blob/e7b5f7dbe98c559b20c0c8c20c0b31a6b197d717/api/types/swarm/swarm.go#L185
-            if state == "inactive":
-                logger.warning(
-                    "docker swarm is inactive on this host; performing `docker swarm init --advertise-addr 127.0.0.1 --listen-addr 127.0.0.1`"
-                )
-                client.swarm.init(advertise_addr="127.0.0.1", listen_addr="127.0.0.1")
-            elif state == "active":
-                break
-            else:
-                logger.notice(  # pyre-fixme
-                    StructuredLogMessage("waiting for docker swarm to become active", state=state)
-                )
-                sleep(2)
-
-        miniwdl_services = [
-            d
-            for d in [s.attrs for s in client.services.list()]
-            if "Spec" in d and "Labels" in d["Spec"] and "miniwdl_run_id" in d["Spec"]["Labels"]
-        ]
-        if miniwdl_services:
-            logger.warning(
-                "docker swarm lists existing miniwdl-related services. This is normal if other miniwdl processes are running concurrently; otherwise, stale state could interfere with this run. To reset it, `docker swarm leave --force`"
-            )
-    finally:
-        client.close()
-
-
 _terminating: Optional[bool] = None
 _terminating_lock: threading.Lock = threading.Lock()
 
@@ -545,3 +508,73 @@ def next(self) -> int:
         with self._lock:
             self._value += 1
             return self._value
+
+
+@export
+def initialize_local_docker(
+    logger: logging.Logger, client: Optional[docker.DockerClient] = None
+) -> Dict[str, int]:
+    client_in = client
+    client = client or docker.from_env()
+    detector = None
+    try:
+        # initialize docker swarm
+        state = "(unknown)"
+        while True:
+            info = client.info()
+            if "Swarm" in info and "LocalNodeState" in info["Swarm"]:
+                state = info["Swarm"]["LocalNodeState"]
+
+            # https://github.com/moby/moby/blob/e7b5f7dbe98c559b20c0c8c20c0b31a6b197d717/api/types/swarm/swarm.go#L185
+            if state == "inactive":
+                logger.warning(
+                    "docker swarm is inactive on this host; performing `docker swarm init --advertise-addr 127.0.0.1 --listen-addr 127.0.0.1`"
+                )
+                client.swarm.init(advertise_addr="127.0.0.1", listen_addr="127.0.0.1")
+            elif state == "active":
+                break
+            else:
+                logger.notice(  # pyre-fixme
+                    StructuredLogMessage("waiting for docker swarm to become active", state=state)
+                )
+                sleep(2)
+
+        miniwdl_services = [
+            d
+            for d in [s.attrs for s in client.services.list()]
+            if "Spec" in d and "Labels" in d["Spec"] and "miniwdl_run_id" in d["Spec"]["Labels"]
+        ]
+        if miniwdl_services:
+            logger.warning(
+                "docker swarm lists existing miniwdl-related services. This is normal if other miniwdl processes are running concurrently; otherwise, stale state could interfere with this run. To reset it, `docker swarm leave --force`"
+            )
+
+        # Detect CPUs & memory available to Docker containers on the local host. These limits may
+        # differ from multiprocessing.cpu_count() and psutil.virtual_memory().total; in particular
+        # on macOS, where Docker containers run in a virtual machine with limited resources.
+        detector = client.containers.run(
+            "alpine:3",
+            name=f"wdl-detector-{os.getpid()}",
+            command=["/bin/ash", "-c", "nproc && free -b | awk '/^Mem:/{print $2}'"],
+            detach=True,
+        )
+        status = detector.wait()
+        assert isinstance(status, dict) and status.get("StatusCode", -1) == 0, str(status)
+        stdout = detector.logs(stdout=True)
+        logger.debug(StructuredLogMessage("docker resource detection", stdout=str(stdout)))
+        stdout = stdout.decode("utf-8").strip().split("\n")
+        assert len(stdout) == 2
+        ans = (int(stdout[0]), int(stdout[1]))
+        logger.info(
+            StructuredLogMessage(
+                "detected host resources available for Docker containers",
+                cpu=ans[0],
+                mem_bytes=ans[1],
+            )
+        )
+        return {"runtime_cpu_max": ans[0], "runtime_memory_max": ans[1]}
+    finally:
+        if detector:
+            detector.remove()
+        if client and not client_in:
+            client.close()