Merge pull request #235 from FNNDSC/shm-size

Implement support for changing SHM_SIZE

README.md

@@ -132,6 +132,7 @@ https://github.com/containers/podman/blob/main/troubleshooting.md#symptom-23
 | `PFCON_SELECTOR`         | label on the pfcon container, may be specified for pman to self-discover `VOLUME_NAME` (default: `org.chrisproject.role=pfcon`) |
 | `CONTAINER_USER`         | Set job container user in the form `UID:GID`, may be a range for random values                                                  | 
 | `ENABLE_HOME_WORKAROUND` | If set to "yes" then set job environment variable `HOME=/tmp`                                                                   |
+| `SHM_SIZE`               | Size of `/dev/shm` in mebibytes. (Supported only in Docker, Podman, and Kubernetes.)                                            |
 | `JOB_LABELS`             | CSV list of key=value pairs, labels to apply to container jobs                                                                  |
 | `JOB_LOGS_TAIL`          | (int) maximum size of job logs                                                                                                  |
 | `IGNORE_LIMITS`          | If set to "yes" then do not set resource limits on container jobs (for making things work without effort)                       |

pman/config.py

@@ -5,6 +5,7 @@
 
 from importlib.metadata import Distribution
 
+from pman.memsize import Memsize
 from pman._helpers import get_storebase_from_docker
 
 pkg = Distribution.from_name(__package__)
@@ -28,6 +29,8 @@ def __init__(self):
         self.IGNORE_LIMITS = env.bool('IGNORE_LIMITS', False)
         self.CONTAINER_USER = env('CONTAINER_USER', None)
         self.ENABLE_HOME_WORKAROUND = env.bool('ENABLE_HOME_WORKAROUND', False)
+        shm_size = env.int('SHM_SIZE', None)
+        self.SHM_SIZE = None if shm_size is None else Memsize(shm_size)
 
         self.CONTAINER_ENV = env('CONTAINER_ENV', 'docker')
         if self.CONTAINER_ENV == 'podman':  # podman is just an alias for docker

pman/dockermgr.py

@@ -59,6 +59,10 @@ def schedule_job(self, image: Image, command: List[str], name: JobName,
         if gid is not None:
             user_spec['group_add'] = [gid]
 
+        shm_size = {}
+        if (s := self.config.get('SHM_SIZE')) is not None:
+            shm_size['shm_size'] = s.as_mb()
+
         return self.__docker.containers.run(
             image=image,
             command=command,
@@ -69,6 +73,7 @@ def schedule_job(self, image: Image, command: List[str], name: JobName,
             labels=self.job_labels,
             **limits,
             **user_spec,
+            **shm_size,
             **volumes
         )
 

pman/kubernetesmgr.py

@@ -192,14 +192,29 @@ def create_job(self, image, command, name, resources_dict, env_l, uid, gid,
             read_only=False
         )
 
+        dshm_volume = []
+        dshm_mount = []
+        if (s := self.config.get('SHM_SIZE')) is not None:
+            dshm_volume.append(k_client.V1Volume(
+                name='dshm',
+                empty_dir=k_client.V1EmptyDirVolumeSource(
+                    medium='Memory',
+                    size_limit=s.as_mib()
+                )
+            ))
+            dshm_mount.append(k_client.V1VolumeMount(
+                mount_path='/dev/shm',
+                name='dshm',
+            ))
+
         container = k_client.V1Container(
             name=name,
             image=image,
             env=env,
             command=command,
             security_context=k_client.V1SecurityContext(**security_context),
             resources=k_client.V1ResourceRequirements(limits=limits),
-            volume_mounts=[volume_mount_inputdir, volume_mount_outputdir]
+            volume_mounts=[volume_mount_inputdir, volume_mount_outputdir, *dshm_mount]
         )
 
         pod_template_metadata = None
@@ -213,7 +228,7 @@ def create_job(self, image, command, name, resources_dict, env_l, uid, gid,
             metadata=pod_template_metadata,
             spec=k_client.V1PodSpec(restart_policy='Never',
                                     containers=[container],
-                                    volumes=[volume],
+                                    volumes=[volume, *dshm_volume],
                                     node_selector=node_selector),
 
         )

pman/memsize.py

@@ -0,0 +1,19 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Memsize:
+    """
+    A quantity of bytes.
+    """
+
+    mebibytes: int
+
+    def as_mb(self) -> str:
+        """Value as megabyte string, ending in 'm'"""
+        mb = int(1.048576 * self.mebibytes)
+        return f'{mb}m'
+
+    def as_mib(self) -> str:
+        """Value as mebibyte string, ending in 'Mi'"""
+        return f'{self.mebibytes}Mi'