Skip to content

Commit

Permalink
Dockerfile
Browse files Browse the repository at this point in the history 
- change user to nonroot
- add the BUILDKIT_SBOM_SCAN_CONTEXT buildarg for proper SBOM creation
Docker/build.py
- add provenance
- add sbom attestation
- some formattíng
  • Loading branch information
@dkuegler
dkuegler committed Feb 19, 2024
1 parent cc7eedc commit 4e45d83
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 34 deletions.
31 changes: 22 additions & 9 deletions Docker/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ ARG FREESURFER_BUILD_IMAGE=build_freesurfer
ARG CONDA_BUILD_IMAGE=build_conda
ARG RUNTIME_BASE_IMAGE=ubuntu:22.04
ARG BUILD_BASE_IMAGE=ubuntu:22.04
# BUILDKIT_SBOM:SCAN_CONTEXT enables buildkit to provide and scan build images
# this is active by default to provide proper SBOM manifests, however, it may also
# include parts that are not part of the distributed image (specifically build image
# parts installed in the build image, but not transfered to the runtime image such as
# git, wget, the miniconda installer, etc.)
ARG BUILDKIT_SBOM_SCAN_CONTEXT=true

## Start with ubuntu base to build the conda base stage
FROM $BUILD_BASE_IMAGE AS build_base
Expand Down Expand Up @@ -87,7 +93,8 @@ COPY ./env/fastsurfer.yml ./Docker/conda_pack.sh ./Docker/install_env.py /instal
# Install conda for gpu

ARG DEBUG=false
RUN python /install/install_env.py -m base -i /install/fastsurfer.yml -o /install/base-env.yml && \
RUN python /install/install_env.py -m base -i /install/fastsurfer.yml \
-o /install/base-env.yml && \
mamba env create -f "/install/base-env.yml" | tee /install/env-create.log ; \
if [ "${DEBUG}" != "true" ]; then \
rm /install/base-env.yml ; \
Expand All @@ -98,8 +105,10 @@ FROM build_common AS build_conda
ARG DEBUG=false
ARG DEVICE=cu118
# install additional packages for cuda/rocm/cpu
RUN python /install/install_env.py -m ${DEVICE} -i /install/fastsurfer.yml -o /install/${DEVICE}-env.yml && \
mamba env update -n "fastsurfer" -f "/install/${DEVICE}-env.yml" | tee /install/env-update.log && \
RUN python /install/install_env.py -m ${DEVICE} -i /install/fastsurfer.yml \
-o /install/${DEVICE}-env.yml && \
mamba env update -n "fastsurfer" -f "/install/${DEVICE}-env.yml" \
| tee /install/env-update.log && \
/install/conda_pack.sh "fastsurfer" && \
echo "DEBUG=$DEBUG\nDEVICE=$DEVICE\n" > /install/build_conda.args ; \
if [ "${DEBUG}" != "true" ]; then \
Expand All @@ -124,7 +133,8 @@ ln -s /venv/bin/python3 /opt/freesurfer/bin/fspython
# =======================================================
# Here, we create references to the requested build image
# =======================================================
# This is needed because COPY --from=<image/stage> does not accept variables as part of the image/stage name
# This is needed because COPY --from=<image/stage> does not accept variables as part of
# the image/stage name
# selected_freesurfer_build_image -> $FREESURFER_BUILD_IMAGE
FROM $FREESURFER_BUILD_IMAGE AS selected_freesurfer_build_image
# selected_conda_build_image -> $CONDA_BUILD_IMAGE
Expand Down Expand Up @@ -169,7 +179,8 @@ SHELL ["/bin/bash", "--login", "-c"]

# Copy fastsurfer venv and pruned freesurfer from build images

# Note, since COPY does not support variables in the --from parameter, so we point to a reference here, and the
# Note, since COPY does not support variables in the --from parameter, so we point to a
# reference here, and the
# seletced_<name>_build_image is a only a reference to $<NAME>_BUILD_IMAGE
COPY --from=selected_freesurfer_build_image /opt/freesurfer /opt/freesurfer
COPY --from=selected_conda_build_image /venv /venv
Expand All @@ -180,15 +191,17 @@ COPY . /fastsurfer/
ENV PYTHONPATH=/fastsurfer:/opt/freesurfer/python/packages:$PYTHONPATH \
FASTSURFER_HOME=/fastsurfer

# Download all remote network checkpoints already, compile all FastSurfer scripts into bytecode and update
# the build file with checkpoints md5sums and pip packages.
# Download all remote network checkpoints already, compile all FastSurfer scripts into
# bytecode and update the build file with checkpoints md5sums and pip packages.
RUN cd /fastsurfer ; python3 FastSurferCNN/download_checkpoints.py --all && \
python3 -m compileall * && \
python3 FastSurferCNN/version.py --sections +git+checkpoints+pip --build_cache BUILD.info -o fullBUILD.info && \
python3 FastSurferCNN/version.py --sections +git+checkpoints+pip \
--build_cache BUILD.info -o fullBUILD.info && \
mv fullBUILD.info BUILD.info

# Set FastSurfer workdir and entrypoint
# the script entrypoint ensures that our conda env is active
USER nonroot
WORKDIR "/fastsurfer"
ENTRYPOINT ["/fastsurfer/Docker/entrypoint.sh","/fastsurfer/run_fastsurfer.sh"]
CMD ["--help"]
Expand All @@ -199,4 +212,4 @@ FROM runtime as runtime_cuda

ENV NVIDIA_VISIBLE_DEVICES=all \
NVIDIA_DRIVER_CAPABILITIES=compute,utility \
NVIDIA_REQUIRE_CUDA="cuda>=8.0"
NVIDIA_REQUIRE_CUDA="cuda>=8.0"
82 changes: 57 additions & 25 deletions Docker/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,39 +285,71 @@ def docker_build_image(
image_name: str,
dockerfile: Path,
working_directory: Optional[Path] = None,
context: Path = ".",
context: Path | str = ".",
dry_run: bool = False,
**kwargs):
logger.info("Building. This starts with sending the build context to the docker daemon, which may take a while...")
**kwargs) -> None:
"""
Build a docker image.
Parameters
----------
image_name : str
Name / target tag of the image.
dockerfile : Path, str
Path to the Dockerfile.
working_directory : Path, str, optional
Path o the working directory to perform the build operation (default: inherit).
context : Path, str, optional
Base path to the context folder to build the docker image from (default: '.').
dry_run : bool, optional
Whether to actually trigger the build, or just print the command to the console
(default: False => actually build).
cache_to : str, optional
Forces usage of buildx over build, use docker build caching as in the --cache-to
argument to docker buildx build.
Other Parameters
----------------
Additional kwargs add additional build flags to the build command in the following
manner: "_" is replaced by "-" in the keyword name and each sequence entry is passed
with its own flag, e.g. `docker_build_image(..., build_arg=["TEST=1", "VAL=2"])` is
translated to `docker [buildx] build ... --build-arg TEST=1 --build-arg VAL=2`.
"""
from itertools import chain, repeat
from subprocess import PIPE
logger.info("Building. This starts with sending the build context to the docker "
"daemon, which may take a while...")
extra_env = {"DOCKER_BUILDKIT": "1"}
from itertools import chain

def to_pair(key, values):
_values = values if isinstance(values, Sequence) and not isinstance(values, (str, bytes)) else [values]
if isinstance(values, Sequence) and isinstance(values, (str, bytes)):
values = [values]
key_dashed = key.replace("_", "-")
return list(chain(*[[f"--{key_dashed}"] + ([] if val is None else [val]) for val in _values]))
# concatenate the --key_dashed value pairs
return list(chain(*zip(repeat(f"--{key_dashed}"), values)))

# needs buildx
buildx = "cache_to" in kwargs
args = ["buildx", "build"] if buildx else ["build"]

if buildx:
Popen = _import_calls(working_directory) # from fastsurfer dir
buildx_test = Popen(["docker", "buildx", "version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).finish()
if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip():
raise RuntimeError(
"Using --cache requires docker buildx, install with 'wget -qO ~/"
".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/"
"releases/download/<version>/buildx-<version>.<platform>'\n"
"e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx "
"https://github.com/docker/buildx/releases/download/v0.11.2/"
"buildx-v0.11.2.linux-amd64'\n"
"You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n"
"See also https://github.com/docker/buildx#manual-download")

args = ["buildx", "build"]

# always use/require buildx (required for sbom and provenance)
Popen = _import_calls(working_directory) # from fastsurfer dir
cmd_exec_args = ["docker", "buildx", "version"]
buildx_test = Popen(cmd_exec_args, stdout=PIPE, stderr=PIPE).finish()
if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip():
raise RuntimeError(
"Using --cache requires docker buildx, install with 'wget -qO ~/"
".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/"
"releases/download/<version>/buildx-<version>.<platform>'\n"
"e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx "
"https://github.com/docker/buildx/releases/download/v0.12.1/"
"buildx-v0.12.1.linux-amd64'\n"
"You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n"
"See also https://github.com/docker/buildx#manual-download"
)
params = [to_pair(*a) for a in kwargs.items()]

args += ["-t", image_name, "-f", str(dockerfile)] + list(chain(*params)) + [str(context)]
args.extend(["--attest", "type=sbom", "--provenance=true"])
args.extend(["-t", image_name, "-f", str(dockerfile)] + list(chain(*params)))
args.append(str(context))
if dry_run:
extra_environment = [f"{k}={v}" for k, v in extra_env.items()]
print(" ".join(extra_environment + ["docker"] + args))
Expand Down

0 comments on commit 4e45d83

Please sign in to comment.