From 4e45d83aa7d3015f848cfc00b48e1ae82d8ec790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20K=C3=BCgler?= Date: Wed, 31 Jan 2024 18:25:03 +0100 Subject: [PATCH] =?UTF-8?q?Dockerfile=20-=20change=20user=20to=20nonroot?= =?UTF-8?q?=20-=20add=20the=20BUILDKIT=5FSBOM=5FSCAN=5FCONTEXT=20buildarg?= =?UTF-8?q?=20for=20proper=20SBOM=20creation=20Docker/build.py=20-=20add?= =?UTF-8?q?=20provenance=20-=20add=20sbom=20attestation=20-=20some=20forma?= =?UTF-8?q?tt=C3=ADng?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Docker/Dockerfile | 31 ++++++++++++------ Docker/build.py | 82 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 79 insertions(+), 34 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index a82603a5..0f43d880 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -48,6 +48,12 @@ ARG FREESURFER_BUILD_IMAGE=build_freesurfer ARG CONDA_BUILD_IMAGE=build_conda ARG RUNTIME_BASE_IMAGE=ubuntu:22.04 ARG BUILD_BASE_IMAGE=ubuntu:22.04 +# BUILDKIT_SBOM:SCAN_CONTEXT enables buildkit to provide and scan build images +# this is active by default to provide proper SBOM manifests, however, it may also +# include parts that are not part of the distributed image (specifically build image +# parts installed in the build image, but not transfered to the runtime image such as +# git, wget, the miniconda installer, etc.) +ARG BUILDKIT_SBOM_SCAN_CONTEXT=true ## Start with ubuntu base to build the conda base stage FROM $BUILD_BASE_IMAGE AS build_base @@ -87,7 +93,8 @@ COPY ./env/fastsurfer.yml ./Docker/conda_pack.sh ./Docker/install_env.py /instal # Install conda for gpu ARG DEBUG=false -RUN python /install/install_env.py -m base -i /install/fastsurfer.yml -o /install/base-env.yml && \ +RUN python /install/install_env.py -m base -i /install/fastsurfer.yml \ + -o /install/base-env.yml && \ mamba env create -f "/install/base-env.yml" | tee /install/env-create.log ; \ if [ "${DEBUG}" != "true" ]; then \ rm /install/base-env.yml ; \ @@ -98,8 +105,10 @@ FROM build_common AS build_conda ARG DEBUG=false ARG DEVICE=cu118 # install additional packages for cuda/rocm/cpu -RUN python /install/install_env.py -m ${DEVICE} -i /install/fastsurfer.yml -o /install/${DEVICE}-env.yml && \ - mamba env update -n "fastsurfer" -f "/install/${DEVICE}-env.yml" | tee /install/env-update.log && \ +RUN python /install/install_env.py -m ${DEVICE} -i /install/fastsurfer.yml \ + -o /install/${DEVICE}-env.yml && \ + mamba env update -n "fastsurfer" -f "/install/${DEVICE}-env.yml" \ + | tee /install/env-update.log && \ /install/conda_pack.sh "fastsurfer" && \ echo "DEBUG=$DEBUG\nDEVICE=$DEVICE\n" > /install/build_conda.args ; \ if [ "${DEBUG}" != "true" ]; then \ @@ -124,7 +133,8 @@ ln -s /venv/bin/python3 /opt/freesurfer/bin/fspython # ======================================================= # Here, we create references to the requested build image # ======================================================= -# This is needed because COPY --from= does not accept variables as part of the image/stage name +# This is needed because COPY --from= does not accept variables as part of +# the image/stage name # selected_freesurfer_build_image -> $FREESURFER_BUILD_IMAGE FROM $FREESURFER_BUILD_IMAGE AS selected_freesurfer_build_image # selected_conda_build_image -> $CONDA_BUILD_IMAGE @@ -169,7 +179,8 @@ SHELL ["/bin/bash", "--login", "-c"] # Copy fastsurfer venv and pruned freesurfer from build images -# Note, since COPY does not support variables in the --from parameter, so we point to a reference here, and the +# Note, since COPY does not support variables in the --from parameter, so we point to a +# reference here, and the # seletced__build_image is a only a reference to $_BUILD_IMAGE COPY --from=selected_freesurfer_build_image /opt/freesurfer /opt/freesurfer COPY --from=selected_conda_build_image /venv /venv @@ -180,15 +191,17 @@ COPY . /fastsurfer/ ENV PYTHONPATH=/fastsurfer:/opt/freesurfer/python/packages:$PYTHONPATH \ FASTSURFER_HOME=/fastsurfer -# Download all remote network checkpoints already, compile all FastSurfer scripts into bytecode and update -# the build file with checkpoints md5sums and pip packages. +# Download all remote network checkpoints already, compile all FastSurfer scripts into +# bytecode and update the build file with checkpoints md5sums and pip packages. RUN cd /fastsurfer ; python3 FastSurferCNN/download_checkpoints.py --all && \ python3 -m compileall * && \ - python3 FastSurferCNN/version.py --sections +git+checkpoints+pip --build_cache BUILD.info -o fullBUILD.info && \ + python3 FastSurferCNN/version.py --sections +git+checkpoints+pip \ + --build_cache BUILD.info -o fullBUILD.info && \ mv fullBUILD.info BUILD.info # Set FastSurfer workdir and entrypoint # the script entrypoint ensures that our conda env is active +USER nonroot WORKDIR "/fastsurfer" ENTRYPOINT ["/fastsurfer/Docker/entrypoint.sh","/fastsurfer/run_fastsurfer.sh"] CMD ["--help"] @@ -199,4 +212,4 @@ FROM runtime as runtime_cuda ENV NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility \ - NVIDIA_REQUIRE_CUDA="cuda>=8.0" + NVIDIA_REQUIRE_CUDA="cuda>=8.0" \ No newline at end of file diff --git a/Docker/build.py b/Docker/build.py index 47b2dd1f..394d0812 100755 --- a/Docker/build.py +++ b/Docker/build.py @@ -285,39 +285,71 @@ def docker_build_image( image_name: str, dockerfile: Path, working_directory: Optional[Path] = None, - context: Path = ".", + context: Path | str = ".", dry_run: bool = False, - **kwargs): - logger.info("Building. This starts with sending the build context to the docker daemon, which may take a while...") + **kwargs) -> None: + """ + Build a docker image. + + Parameters + ---------- + image_name : str + Name / target tag of the image. + dockerfile : Path, str + Path to the Dockerfile. + working_directory : Path, str, optional + Path o the working directory to perform the build operation (default: inherit). + context : Path, str, optional + Base path to the context folder to build the docker image from (default: '.'). + dry_run : bool, optional + Whether to actually trigger the build, or just print the command to the console + (default: False => actually build). + cache_to : str, optional + Forces usage of buildx over build, use docker build caching as in the --cache-to + argument to docker buildx build. + + Other Parameters + ---------------- + Additional kwargs add additional build flags to the build command in the following + manner: "_" is replaced by "-" in the keyword name and each sequence entry is passed + with its own flag, e.g. `docker_build_image(..., build_arg=["TEST=1", "VAL=2"])` is + translated to `docker [buildx] build ... --build-arg TEST=1 --build-arg VAL=2`. + """ + from itertools import chain, repeat + from subprocess import PIPE + logger.info("Building. This starts with sending the build context to the docker " + "daemon, which may take a while...") extra_env = {"DOCKER_BUILDKIT": "1"} - from itertools import chain def to_pair(key, values): - _values = values if isinstance(values, Sequence) and not isinstance(values, (str, bytes)) else [values] + if isinstance(values, Sequence) and isinstance(values, (str, bytes)): + values = [values] key_dashed = key.replace("_", "-") - return list(chain(*[[f"--{key_dashed}"] + ([] if val is None else [val]) for val in _values])) + # concatenate the --key_dashed value pairs + return list(chain(*zip(repeat(f"--{key_dashed}"), values))) # needs buildx - buildx = "cache_to" in kwargs - args = ["buildx", "build"] if buildx else ["build"] - - if buildx: - Popen = _import_calls(working_directory) # from fastsurfer dir - buildx_test = Popen(["docker", "buildx", "version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).finish() - if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip(): - raise RuntimeError( - "Using --cache requires docker buildx, install with 'wget -qO ~/" - ".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/" - "releases/download//buildx-.'\n" - "e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx " - "https://github.com/docker/buildx/releases/download/v0.11.2/" - "buildx-v0.11.2.linux-amd64'\n" - "You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n" - "See also https://github.com/docker/buildx#manual-download") - + args = ["buildx", "build"] + + # always use/require buildx (required for sbom and provenance) + Popen = _import_calls(working_directory) # from fastsurfer dir + cmd_exec_args = ["docker", "buildx", "version"] + buildx_test = Popen(cmd_exec_args, stdout=PIPE, stderr=PIPE).finish() + if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip(): + raise RuntimeError( + "Using --cache requires docker buildx, install with 'wget -qO ~/" + ".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/" + "releases/download//buildx-.'\n" + "e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx " + "https://github.com/docker/buildx/releases/download/v0.12.1/" + "buildx-v0.12.1.linux-amd64'\n" + "You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n" + "See also https://github.com/docker/buildx#manual-download" + ) params = [to_pair(*a) for a in kwargs.items()] - - args += ["-t", image_name, "-f", str(dockerfile)] + list(chain(*params)) + [str(context)] + args.extend(["--attest", "type=sbom", "--provenance=true"]) + args.extend(["-t", image_name, "-f", str(dockerfile)] + list(chain(*params))) + args.append(str(context)) if dry_run: extra_environment = [f"{k}={v}" for k, v in extra_env.items()] print(" ".join(extra_environment + ["docker"] + args))