From 4e45d83aa7d3015f848cfc00b48e1ae82d8ec790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20K=C3=BCgler?= Date: Wed, 31 Jan 2024 18:25:03 +0100 Subject: [PATCH 1/3] =?UTF-8?q?Dockerfile=20-=20change=20user=20to=20nonro?= =?UTF-8?q?ot=20-=20add=20the=20BUILDKIT=5FSBOM=5FSCAN=5FCONTEXT=20buildar?= =?UTF-8?q?g=20for=20proper=20SBOM=20creation=20Docker/build.py=20-=20add?= =?UTF-8?q?=20provenance=20-=20add=20sbom=20attestation=20-=20some=20forma?= =?UTF-8?q?tt=C3=ADng?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Docker/Dockerfile | 31 ++++++++++++------ Docker/build.py | 82 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 79 insertions(+), 34 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index a82603a5..0f43d880 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -48,6 +48,12 @@ ARG FREESURFER_BUILD_IMAGE=build_freesurfer ARG CONDA_BUILD_IMAGE=build_conda ARG RUNTIME_BASE_IMAGE=ubuntu:22.04 ARG BUILD_BASE_IMAGE=ubuntu:22.04 +# BUILDKIT_SBOM:SCAN_CONTEXT enables buildkit to provide and scan build images +# this is active by default to provide proper SBOM manifests, however, it may also +# include parts that are not part of the distributed image (specifically build image +# parts installed in the build image, but not transfered to the runtime image such as +# git, wget, the miniconda installer, etc.) +ARG BUILDKIT_SBOM_SCAN_CONTEXT=true ## Start with ubuntu base to build the conda base stage FROM $BUILD_BASE_IMAGE AS build_base @@ -87,7 +93,8 @@ COPY ./env/fastsurfer.yml ./Docker/conda_pack.sh ./Docker/install_env.py /instal # Install conda for gpu ARG DEBUG=false -RUN python /install/install_env.py -m base -i /install/fastsurfer.yml -o /install/base-env.yml && \ +RUN python /install/install_env.py -m base -i /install/fastsurfer.yml \ + -o /install/base-env.yml && \ mamba env create -f "/install/base-env.yml" | tee /install/env-create.log ; \ if [ "${DEBUG}" != "true" ]; then \ rm /install/base-env.yml ; \ @@ -98,8 +105,10 @@ FROM build_common AS build_conda ARG DEBUG=false ARG DEVICE=cu118 # install additional packages for cuda/rocm/cpu -RUN python /install/install_env.py -m ${DEVICE} -i /install/fastsurfer.yml -o /install/${DEVICE}-env.yml && \ - mamba env update -n "fastsurfer" -f "/install/${DEVICE}-env.yml" | tee /install/env-update.log && \ +RUN python /install/install_env.py -m ${DEVICE} -i /install/fastsurfer.yml \ + -o /install/${DEVICE}-env.yml && \ + mamba env update -n "fastsurfer" -f "/install/${DEVICE}-env.yml" \ + | tee /install/env-update.log && \ /install/conda_pack.sh "fastsurfer" && \ echo "DEBUG=$DEBUG\nDEVICE=$DEVICE\n" > /install/build_conda.args ; \ if [ "${DEBUG}" != "true" ]; then \ @@ -124,7 +133,8 @@ ln -s /venv/bin/python3 /opt/freesurfer/bin/fspython # ======================================================= # Here, we create references to the requested build image # ======================================================= -# This is needed because COPY --from= does not accept variables as part of the image/stage name +# This is needed because COPY --from= does not accept variables as part of +# the image/stage name # selected_freesurfer_build_image -> $FREESURFER_BUILD_IMAGE FROM $FREESURFER_BUILD_IMAGE AS selected_freesurfer_build_image # selected_conda_build_image -> $CONDA_BUILD_IMAGE @@ -169,7 +179,8 @@ SHELL ["/bin/bash", "--login", "-c"] # Copy fastsurfer venv and pruned freesurfer from build images -# Note, since COPY does not support variables in the --from parameter, so we point to a reference here, and the +# Note, since COPY does not support variables in the --from parameter, so we point to a +# reference here, and the # seletced__build_image is a only a reference to $_BUILD_IMAGE COPY --from=selected_freesurfer_build_image /opt/freesurfer /opt/freesurfer COPY --from=selected_conda_build_image /venv /venv @@ -180,15 +191,17 @@ COPY . /fastsurfer/ ENV PYTHONPATH=/fastsurfer:/opt/freesurfer/python/packages:$PYTHONPATH \ FASTSURFER_HOME=/fastsurfer -# Download all remote network checkpoints already, compile all FastSurfer scripts into bytecode and update -# the build file with checkpoints md5sums and pip packages. +# Download all remote network checkpoints already, compile all FastSurfer scripts into +# bytecode and update the build file with checkpoints md5sums and pip packages. RUN cd /fastsurfer ; python3 FastSurferCNN/download_checkpoints.py --all && \ python3 -m compileall * && \ - python3 FastSurferCNN/version.py --sections +git+checkpoints+pip --build_cache BUILD.info -o fullBUILD.info && \ + python3 FastSurferCNN/version.py --sections +git+checkpoints+pip \ + --build_cache BUILD.info -o fullBUILD.info && \ mv fullBUILD.info BUILD.info # Set FastSurfer workdir and entrypoint # the script entrypoint ensures that our conda env is active +USER nonroot WORKDIR "/fastsurfer" ENTRYPOINT ["/fastsurfer/Docker/entrypoint.sh","/fastsurfer/run_fastsurfer.sh"] CMD ["--help"] @@ -199,4 +212,4 @@ FROM runtime as runtime_cuda ENV NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility \ - NVIDIA_REQUIRE_CUDA="cuda>=8.0" + NVIDIA_REQUIRE_CUDA="cuda>=8.0" \ No newline at end of file diff --git a/Docker/build.py b/Docker/build.py index 47b2dd1f..394d0812 100755 --- a/Docker/build.py +++ b/Docker/build.py @@ -285,39 +285,71 @@ def docker_build_image( image_name: str, dockerfile: Path, working_directory: Optional[Path] = None, - context: Path = ".", + context: Path | str = ".", dry_run: bool = False, - **kwargs): - logger.info("Building. This starts with sending the build context to the docker daemon, which may take a while...") + **kwargs) -> None: + """ + Build a docker image. + + Parameters + ---------- + image_name : str + Name / target tag of the image. + dockerfile : Path, str + Path to the Dockerfile. + working_directory : Path, str, optional + Path o the working directory to perform the build operation (default: inherit). + context : Path, str, optional + Base path to the context folder to build the docker image from (default: '.'). + dry_run : bool, optional + Whether to actually trigger the build, or just print the command to the console + (default: False => actually build). + cache_to : str, optional + Forces usage of buildx over build, use docker build caching as in the --cache-to + argument to docker buildx build. + + Other Parameters + ---------------- + Additional kwargs add additional build flags to the build command in the following + manner: "_" is replaced by "-" in the keyword name and each sequence entry is passed + with its own flag, e.g. `docker_build_image(..., build_arg=["TEST=1", "VAL=2"])` is + translated to `docker [buildx] build ... --build-arg TEST=1 --build-arg VAL=2`. + """ + from itertools import chain, repeat + from subprocess import PIPE + logger.info("Building. This starts with sending the build context to the docker " + "daemon, which may take a while...") extra_env = {"DOCKER_BUILDKIT": "1"} - from itertools import chain def to_pair(key, values): - _values = values if isinstance(values, Sequence) and not isinstance(values, (str, bytes)) else [values] + if isinstance(values, Sequence) and isinstance(values, (str, bytes)): + values = [values] key_dashed = key.replace("_", "-") - return list(chain(*[[f"--{key_dashed}"] + ([] if val is None else [val]) for val in _values])) + # concatenate the --key_dashed value pairs + return list(chain(*zip(repeat(f"--{key_dashed}"), values))) # needs buildx - buildx = "cache_to" in kwargs - args = ["buildx", "build"] if buildx else ["build"] - - if buildx: - Popen = _import_calls(working_directory) # from fastsurfer dir - buildx_test = Popen(["docker", "buildx", "version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).finish() - if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip(): - raise RuntimeError( - "Using --cache requires docker buildx, install with 'wget -qO ~/" - ".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/" - "releases/download//buildx-.'\n" - "e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx " - "https://github.com/docker/buildx/releases/download/v0.11.2/" - "buildx-v0.11.2.linux-amd64'\n" - "You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n" - "See also https://github.com/docker/buildx#manual-download") - + args = ["buildx", "build"] + + # always use/require buildx (required for sbom and provenance) + Popen = _import_calls(working_directory) # from fastsurfer dir + cmd_exec_args = ["docker", "buildx", "version"] + buildx_test = Popen(cmd_exec_args, stdout=PIPE, stderr=PIPE).finish() + if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip(): + raise RuntimeError( + "Using --cache requires docker buildx, install with 'wget -qO ~/" + ".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/" + "releases/download//buildx-.'\n" + "e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx " + "https://github.com/docker/buildx/releases/download/v0.12.1/" + "buildx-v0.12.1.linux-amd64'\n" + "You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n" + "See also https://github.com/docker/buildx#manual-download" + ) params = [to_pair(*a) for a in kwargs.items()] - - args += ["-t", image_name, "-f", str(dockerfile)] + list(chain(*params)) + [str(context)] + args.extend(["--attest", "type=sbom", "--provenance=true"]) + args.extend(["-t", image_name, "-f", str(dockerfile)] + list(chain(*params))) + args.append(str(context)) if dry_run: extra_environment = [f"{k}={v}" for k, v in extra_env.items()] print(" ".join(extra_environment + ["docker"] + args)) From 9123515bc38180fabf6c49cfaad7cf859c3e1ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20K=C3=BCgler?= Date: Mon, 19 Feb 2024 19:24:08 +0100 Subject: [PATCH 2/3] Docker/Dockerfile - add FREESURFER_URL build-arg and pass the URL to install_fs_pruned.sh Docker/install_fs_pruned.sh - add option to download FreeSurfer froma different URL - optimize upx option for multiple threads - reorder so upx runs before link are created Docker/build.py - add --attest argument - change building logic, so it works with docker-container - add attestation logic --- Docker/Dockerfile | 4 +- Docker/build.py | 176 ++++++++++++++++++++++++++++-------- Docker/install_fs_pruned.sh | 87 ++++++++++++++---- 3 files changed, 214 insertions(+), 53 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 0f43d880..29379615 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -123,8 +123,10 @@ FROM build_base AS build_freesurfer COPY ./Docker/install_fs_pruned.sh /install/ SHELL ["/bin/bash", "--login", "-c"] +ARG FREESURFER_URL=default + # install freesurfer and point to new python location -RUN /install/install_fs_pruned.sh /opt --upx && \ +RUN /install/install_fs_pruned.sh /opt --upx --url $FREESURFER_URL && \ rm /opt/freesurfer/bin/fspython && \ rm -R /install && \ ln -s /venv/bin/python3 /opt/freesurfer/bin/fspython diff --git a/Docker/build.py b/Docker/build.py index 394d0812..d86b4afd 100755 --- a/Docker/build.py +++ b/Docker/build.py @@ -82,12 +82,14 @@ def __import(file: Path, name: str, *tokens: str, **rename_tokens: str): def docker_image(arg) -> str: - """Returns a str with the image. + """ + Returns a str with the image. Raises ====== ArgumentTypeError - if it is not a valid docker image.""" + if it is not a valid docker image. + """ from re import match # regex from https://stackoverflow.com/questions/39671641/regex-to-parse-docker-tag pattern = r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(? str: if match(pattern, arg): return arg else: - raise argparse.ArgumentTypeError(f"The image '{arg}' does not look like a " - f"valid image name.") + raise argparse.ArgumentTypeError( + f"The image '{arg}' does not look like a valid image name." + ) def target(arg) -> Target: @@ -112,7 +115,8 @@ def target(arg) -> Target: return cast(Target, arg) else: raise argparse.ArgumentTypeError( - f"target must be one of {', '.join(get_args(Target))}, but was {arg}.") + f"target must be one of {', '.join(get_args(Target))}, but was {arg}." + ) class CacheSpec: @@ -204,7 +208,14 @@ def make_parser() -> argparse.ArgumentParser: type=docker_image, dest="image_tag", metavar="image[:tag]", - help="""tag build stage/target as [:]""") + help="""tag build stage/target as [:]""", + ) + parser.add_argument( + "--attest", + action="store_true", + help="add sbom and provenance attestation (requires docker-container buildkit " + "builder created with 'docker buildx create')", + ) parser.add_argument( "--target", default="runtime", @@ -214,28 +225,47 @@ def make_parser() -> argparse.ArgumentParser: help=f"""target to build (from list of targets below, defaults to runtime):
- build_conda: "finished" conda build image
- build_freesurfer: "finished" freesurfer build image
- - runtime: final fastsurfer runtime image""") + - runtime: final fastsurfer runtime image""", + ) parser.add_argument( "--rm", action="store_true", - help="disables caching, i.e. removes all intermediate images.") + help="disables caching, i.e. removes all intermediate images.", + ) + cache_kwargs = {} + if "FASTSURFER_BUILD_CACHE" in os.environ: + try: + cache_kwargs = { + "default": CacheSpec(os.environ["FASTSURFER_BUILD_CACHE"]) + } + except ValueError as e: + logger.warning( + f"ERROR while parsing the environment variable 'FASTSURFER_BUILD_CACHE' " + f"{os.environ['FASTSURFER_BUILD_CACHE']} (ignoring this environment " + f"variable): {e.args[0]}" + ) parser.add_argument( "--cache", type=CacheSpec, - help="""cache as defined in https://docs.docker.com/build/cache/backends/ - (using --cache-to syntax, parameters are automatically filtered for use - in --cache-to and --cache-from), e.g.: - --cache type=registry,ref=server/fastbuild,mode=max.""") + help=f"""cache as defined in https://docs.docker.com/build/cache/backends/ + (using --cache-to syntax, parameters are automatically filtered for use + in --cache-to and --cache-from), e.g.: + --cache type=registry,ref=server/fastbuild,mode=max. + Will default to the environment variable FASTSURFER_BUILD_CACHE: + {cache_kwargs.get('default', 'N/A')}""", + **cache_kwargs, + ) parser.add_argument( "--dry_run", "--print", action="store_true", help="Instead of starting processes, write the commands to stdout, so they can " - "be dry_run with 'build.py ... --dry_run | bash'.") + "be dry_run with 'build.py ... --dry_run | bash'.", + ) parser.add_argument( "--tag_dev", action="store_true", - help="Also tag the resulting image as 'fastsurfer:dev'." + help="Also tag the resulting image as 'fastsurfer:dev'.", ) expert = parser.add_argument_group('Expert options') @@ -281,12 +311,56 @@ def red(skk): return "\033[91m {}\033[00m" .format(skk) +def get_builder(Popen, require_builder_type: str) -> tuple[bool, str]: + """Get the builder to build the fastsurfer image.""" + from subprocess import PIPE + from re import compile + buildx_binfo = Popen(["docker", "buildx", "ls"], stdout=PIPE, stderr=PIPE).finish() + header, *lines = buildx_binfo.out_str("utf-8").strip().split("\n") + header_pattern = compile("\\S+\\s*") + fields = {} + pos = 0 + while pos < len(header) and (match := header_pattern.search(header, pos)): + start, pos = match.span() + fields[match.group().strip()] = slice(start, pos) + builders = {line[fields["NAME/NODE"]]: line[fields["DRIVER/ENDPOINT"]] + for line in lines if not line.startswith(" ")} + builders = {key.strip(): value.strip() for key, value in builders.items()} + default_builders = [name for name in builders.keys() if name.endswith("*")] + if len(default_builders) != 1: + raise RuntimeError("Could not find default builder of buildx") + default_builder = default_builders[0][:-1].strip() + builders[default_builder] = builders[default_builders[0]] + del builders[default_builders[0]] + cannot_use_default_builder = ( + require_builder_type and builders[default_builder] != require_builder_type + ) + if cannot_use_default_builder: + # if the default builder is a docker builder (which does not support + for builder in builders.keys(): + if (builder.startswith("fastsurfer") and + builders[builder] == require_builder_type): + default_builder = builder + break + if builders[default_builder] != require_builder_type: + # did not find an appropriate builder + raise RuntimeError( + "Could not find an appropriate builder from the current builder " + "(see docker buildx use) or builders named fastsurfer* (searching for " + f"a builder of type {require_builder_type}, docker " + "builders may not be supported with the selected export settings. " + "Create builder with 'docker buildx create --name fastsurfer'." + ) + return not cannot_use_default_builder, default_builder + + def docker_build_image( image_name: str, dockerfile: Path, working_directory: Optional[Path] = None, context: Path | str = ".", dry_run: bool = False, + attestation: bool = False, **kwargs) -> None: """ Build a docker image. @@ -307,9 +381,9 @@ def docker_build_image( cache_to : str, optional Forces usage of buildx over build, use docker build caching as in the --cache-to argument to docker buildx build. + attestation : bool, default=False + Whether to create sbom and provenance attestation - Other Parameters - ---------------- Additional kwargs add additional build flags to the build command in the following manner: "_" is replaced by "-" in the keyword name and each sequence entry is passed with its own flag, e.g. `docker_build_image(..., build_arg=["TEST=1", "VAL=2"])` is @@ -321,6 +395,11 @@ def docker_build_image( "daemon, which may take a while...") extra_env = {"DOCKER_BUILDKIT": "1"} + from shutil import which + docker_cmd = which("docker") + if docker_cmd is None: + raise FileNotFoundError("Could not locate the docker executable") + def to_pair(key, values): if isinstance(values, Sequence) and isinstance(values, (str, bytes)): values = [values] @@ -333,32 +412,48 @@ def to_pair(key, values): # always use/require buildx (required for sbom and provenance) Popen = _import_calls(working_directory) # from fastsurfer dir - cmd_exec_args = ["docker", "buildx", "version"] - buildx_test = Popen(cmd_exec_args, stdout=PIPE, stderr=PIPE).finish() - if "'buildx' is not a docker command" in buildx_test.err_str('utf-8').strip(): - raise RuntimeError( - "Using --cache requires docker buildx, install with 'wget -qO ~/" - ".docker/cli-plugins/docker-buildx https://github.com/docker/buildx/" - "releases/download//buildx-.'\n" - "e.g. 'wget -qO ~/.docker/cli-plugins/docker-buildx " - "https://github.com/docker/buildx/releases/download/v0.12.1/" - "buildx-v0.12.1.linux-amd64'\n" - "You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n" - "See also https://github.com/docker/buildx#manual-download" + if attestation or \ + any(kwargs.get(f"cache_{c}", "inline") != "inline" for c in ("to", "from")): + buildx_test = Popen( + [docker_cmd, "buildx", "version"], + stdout=PIPE, + stderr=PIPE, + ).finish() + if "'buildx' is not a docker command" in buildx_test.err_str("utf-8").strip(): + wget_cmd = ( + "wget -qO ~/.docker/cli-plugins/docker-buildx https://github.com/docker" + "/buildx/releases/download/{0:s}/buildx-{0:s}.{1:s}" + ) + raise RuntimeError( + f"Using --cache or attestation requires docker buildx, install with " + f"'{wget_cmd % ('', '')}'\n" + f"e.g. '{wget_cmd % ('v0.12.1', 'linux-amd64')}\n" + f"You may need to 'chmod +x ~/.docker/cli-plugins/docker-buildx'\n" + f"See also https://github.com/docker/buildx#manual-download" + ) + + if not attestation: + # tag image_name in local registry (simple standard case) + args.extend(["--output", f"type=image,name={image_name}"]) + else: + # want to create sbom and provenance manifests, so needs to use a + # docker-container builder + args.extend(["--attest", "type=sbom", "--provenance=true"]) + can_use_default_builder, alternative_builder = get_builder( + Popen, + "docker-container", ) + if not can_use_default_builder: + args.extend(["--builder", alternative_builder]) + args.extend(["--output", f"type=docker,name={image_name}", "--load"]) + args.extend(("-t", image_name)) params = [to_pair(*a) for a in kwargs.items()] - args.extend(["--attest", "type=sbom", "--provenance=true"]) - args.extend(["-t", image_name, "-f", str(dockerfile)] + list(chain(*params))) + args.extend(["-f", str(dockerfile)] + list(chain(*params))) args.append(str(context)) if dry_run: extra_environment = [f"{k}={v}" for k, v in extra_env.items()] print(" ".join(extra_environment + ["docker"] + args)) else: - from shutil import which - docker_cmd = which("docker") - if docker_cmd is None: - raise FileNotFoundError("Could not locate the docker executable") - Popen = _import_calls(working_directory) # from fastsurfer dir env = dict(os.environ) env.update(extra_env) with Popen([docker_cmd] + args + ["--progress=plain"], @@ -434,6 +529,14 @@ def main( if not bool(image_tag): image_tag = f"fastsurfer:{version_tag}{image_suffix}".replace("+", "_") + attestation = bool(keywords.get("attest")) + if not attestation: + # only attestation requires and actively changes to a docker-container driver + if cache is not None and cache.type != "inline": + return ("The docker build interface only support caching inline, i.e. " + "--cache type=inline. Use --save_docker or --save_oci for other " + "caching drivers.") + if tag_dev: kwargs["tag"] = f"fastsurfer:dev{image_suffix}" @@ -449,7 +552,8 @@ def main( working_directory=fastsurfer_home, context=fastsurfer_home, dry_run=dry_run, - **kwargs + attestation=attestation, + **kwargs, ) except RuntimeError as e: return e.args[0] diff --git a/Docker/install_fs_pruned.sh b/Docker/install_fs_pruned.sh index c8bd5ca5..7367834f 100755 --- a/Docker/install_fs_pruned.sh +++ b/Docker/install_fs_pruned.sh @@ -15,21 +15,41 @@ fslink="https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/7.4.1/freesurfer-linux-ubuntu22_amd64-7.4.1.tar.gz" -if [ "$#" -lt 1 ]; then +if [[ "$#" -lt 1 ]]; then echo - echo "Usage: install_fs_prunded install_dir <--upx>" + echo "Usage: install_fs_prunded install_dir [--upx] [--url freesurfer_download_url]" echo echo "--upx is optional, if passed, fs/bin will be packed" - echo + echo "--url is optional, if passed, freesurfer will be downloaded from it instead of $fslink" + echo exit 2 fi - where=/opt -if [ "$#" -ge 1 ]; then +if [[ "$#" -ge 1 ]]; then where=$1 + shift fi +upx="false" +while [[ "$#" -ge 1 ]]; do + lowercase=$(echo "$1" | tr '[:upper:]' '[:lower:]') + case $lowercase in + --upx) + upx="true" + shift + ;; + --url) + if [[ "$2" != "default" ]]; then fslink=$2; fi + shift + shift + ;; + *) + echo "Invalid argument $1" + exit 1 + ;; + esac +done fss=$where/fs-tmp fsd=$where/freesurfer echo @@ -41,6 +61,42 @@ echo "$fslink" echo +function run_parallel () +{ + # param 1 num_parallel_processes + # param 2 command (printf string) + # param 3 how many entries to consume from $@ per "run" + # param ... parameters to format, ie. we are executing $(printf $command $@...) + i=0 + pids=() + num_parallel_processes=$1 + command=$2 + num=$3 + shift + shift + shift + args=("$@") + j=0 + while [[ "$j" -lt "${#args}" ]] + do + cmd=$(printf "$command" "${args[@]:$j:$num}") + j=$((j + num)) + $cmd & + pids=("${pids[@]}" "$!") + i=$((i + 1)) + if [[ "$i" -ge "$num_parallel_processes" ]] + then + wait "${pids[0]}" + pids=("${pids[@]:1}") + fi + done + for pid in "${pids[@]}" + do + wait "$pid" + done +} + + # get Freesurfer and upack (some of it) echo "Downloading FS and unpacking portions ..." wget --no-check-certificate -qO- $fslink | tar zxv --no-same-owner -C $where \ @@ -338,6 +394,14 @@ do cp -r $fss/$file $fsd/$file done +# pack if desired with upx (do this before adding all the links +if [[ "$upx" == "true" ]] ; then + echo "finding executables in $fsd/bin/..." + exe=$(find $fsd/bin -exec file {} \; | grep ELF | cut -d: -f1) + echo "packing $fsd/bin/ executables (this can take a while) ..." + run_parallel 8 "upx -9 %s %s %s %s" 4 $exe +fi + # Modify fsbindings Python package to allow calling scripts like asegstats2table directly: echo "from . import legacy" > "$fsd/python/packages/fsbindings/__init__.py" @@ -404,7 +468,7 @@ do done # use our python (not really needed in recon-all anyway) -p3=`which python3` +p3=$(which python3) if [ "$p3" == "" ]; then echo "No python3 found, please install first!" echo @@ -413,13 +477,4 @@ fi ln -s $p3 $fsd/bin/fspython #cleanup -rm -rf $fss - -# pack if desired with upx -if [ "$#" -ge 2 ]; then - if [ "${2^^}" == "--UPX" ] ; then - echo "packing $fsd/bin/ executables (this can take a while) ..." - exe=`find $fsd/bin -exec file {} \; | grep ELF | cut -d: -f1` - upx -9 $exe - fi -fi +rm -rf $fss \ No newline at end of file From 0f08ecd38b397d9fefb60ed485a005803b122897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20K=C3=BCgler?= Date: Fri, 23 Feb 2024 11:54:16 +0100 Subject: [PATCH 3/3] build.py - Add action push (no containerd requirement for attestation build) - TODO: FastSurfer/FreeSurfer are not found by the scanner and thus are not in the SBOM Docker/README.md - Add documentation on how to build with attestation --- Docker/Dockerfile | 14 +++++++++++++ Docker/README.md | 51 ++++++++++++++++++++++++++++++++++++++++++++--- Docker/build.py | 33 +++++++++++++++++++++++------- 3 files changed, 88 insertions(+), 10 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 29379615..0bf8168f 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -201,6 +201,20 @@ RUN cd /fastsurfer ; python3 FastSurferCNN/download_checkpoints.py --all && \ --build_cache BUILD.info -o fullBUILD.info && \ mv fullBUILD.info BUILD.info +# TODO: SBOM info of FastSurfer and FreeSurfer are missing, it is unclear how to add +# those at the moment, as the buildscanner syft does not find simple package.json +# or pyproject.toml files right now. The easiest option seems to be to "setup" +# fastsurfer and freesurfer via pip instll. +#ENV BUILDKIT_SCAN_SOURCE_EXTRAS="/fastsurfer" +#ARG BUILDKIT_SCAN_SOURCE_EXTRAS="/fastsurfer" +#RUN < /fastsurfer/package.json +#{ +# "name": "fastsurfer", +# "version": "$(python3 FastSurferCNN/version.py)", +# "author": "David Kügler " +#} +#EOF + # Set FastSurfer workdir and entrypoint # the script entrypoint ensures that our conda env is active USER nonroot diff --git a/Docker/README.md b/Docker/README.md index 4d978dca..8b027045 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -25,10 +25,13 @@ docker run --gpus all -v /home/user/my_mri_data:/data \ * `-v`: This commands mount your data, output and directory with the FreeSurfer license file into the docker container. Inside the container these are visible under the name following the colon (in this case /data, /output, and /fs_license). * `--rm`: The flag takes care of removing the container once the analysis finished. * `-d`: This is optional. You can add this flag to run in detached mode (no screen output and you return to shell) -* `--user $(id -u):$(id -g)`: This part automatically runs the container with your group- (id -g) and user-id (id -u). All generated files will then belong to the specified user. Without the flag, the docker container will be run as root which is strongly discouraged. +* `--user $(id -u):$(id -g)`: Run the container with your account (your user-id and group-id), which are determined by `$(id -u)` and `$(id -g)`, respectively. Running the docker container as root `-u 0:0` is strongly discouraged. + +##### Advanced Docker Flags +* `--group-add `: If additional user groups are required to access files, additional groups may be added via `--group-add [,...]` or `--group-add $(id -G )`. ##### FastSurfer Flags -* The `--fs_license` points to your FreeSurfer license which needs to be available on your computer in the my_fs_license_dir that was mapped above. +* The `--fs_license` points to your FreeSurfer license which needs to be available on your computer in the `my_fs_license_dir` that was mapped above. * The `--t1` points to the t1-weighted MRI image to analyse (full path, with mounted name inside docker: /home/user/my_mri_data => /data) * The `--sid` is the subject ID name (output folder name) * The `--sd` points to the output directory (its mounted name inside docker: /home/user/my_fastsurfer_analysis => /output) @@ -69,7 +72,10 @@ The build script `build.py` supports additional args, targets and options, see ` Note, that the build script's main function is to select parameters for build args, but also create the FastSurfer-root/BUILD.info file, which will be used by FastSurfer to document the version (including git hash of the docker container). This BUILD.info file must exist for the docker build to be successful. In general, if you specify `--dry_run` the command will not be executed but sent to stdout, so you can run `python build.py --device cuda --dry_run | bash` as well. Note, that build.py uses some dependencies from FastSurfer, so you will need to set the PYTHONPATH environment variable to the FastSurfer root (include of `FastSurferCNN` must be possible) and we only support Python 3.10. -By default, the build script will tag your image as "fastsurfer:{version_tag}[-{device}]", where {version_tag} is {version-identifer from pyproject.toml}_{current git-hash} and {device} is the value to --device (and omitted for cuda), but a custom tag can be specified by `--tag {tag_name}`. +By default, the build script will tag your image as `"fastsurfer:{version_tag}[-{device}]"`, where `{version_tag}` is `{version-identifer from pyproject.toml}_{current git-hash}` and `{device}` is the value to `--device` (and omitted for cuda), but a custom tag can be specified by `--tag {tag_name}`. + +#### BuildKit +Note, we recommend using BuildKit to build docker images (e.g. `DOCKER_BUILDKIT=1` -- the build.py script already always adds this). To install BuildKit, run `wget -qO ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download//buildx-.`, for example `wget -qO ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.12.1/buildx-v0.12.1.linux-amd64`. See also https://github.com/docker/buildx#manual-download. ### Example 1: Build GPU FastSurfer Image @@ -156,3 +162,42 @@ docker run --rm --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ --sid subjectX --sd /output ``` +## Build docker image with attestation and provenance + +To build a docker image with attestation and provenance, i.e. Software Bill Of Materials (SBOM) information, several requirements have to be met: + +1. The image must be built with version v0.11+ of BuildKit (we recommend you [install BuildKit](#buildkit) independent of attestation). +2. You must configure a docker-container builder in buildx (`docker buildx create --use --bootstrap --name fastsurfer-bctx --driver docker-container`). Here, you can add additional configuration options such as safe registries to the builder configuration (add `--config /etc/buildkitd.toml`). + ```toml + root = "/path/to/data/for/buildkit" + [worker.containerd] + gckeepstorage=9000 + [[worker.containerd.gcpolicy]] + keepBytes = 512000000 + keepDuration = 172800 + filters = [ "type==source.local", "type==exec.cachemount", "type==source.git.checkout"] + [[worker.containerd.gcpolicy]] + all = true + keepBytes = 1024000000 + # settings to push to a "local", registry with self-signed certificates + # see for example https://tech.paulcz.net/2016/01/secure-docker-with-tls/ https://github.com/paulczar/omgwtfssl + [registry."host:5000"] + ca=["/path/to/registry/ssl/ca.pem"] + [[registry."landau.dzne.ds:5000".keypair]] + key="/path/to/registry/ssl/key.pem" + cert="/path/to/registry/ssl/cert.pem" + ``` +3. Attestation files are not supported by the standard docker image storage driver. Therefore, images cannot be tested locally. + There are two solutions to this limitation. + 1. Directly push to the registry: + Add `--action push` to the build script (the default is `--action load`, which loads the created image into the current docker context, and for the image name, also add the registry name. For example `... python Docker/build.py ... --attest --action push --tag docker.io//fastsurfer:latest`. + 2. [Install the containerd image storage driver](https://docs.docker.com/storage/containerd/#enable-containerd-image-store-on-docker-engine), which supports attestation: To implement this on Linux, make sure your docker daemon config file `/etc/docker/daemon.json` includes + ```json + { + "features": { + "containerd-snapshotter": true + } + } + ``` + Also note, that the image storage location with containerd is not defined by the docker config file `/etc/docker/daemon.json`, but by the containerd config `/etc/containerd/config.toml`, which will likely not exist. You can [create a default config](https://github.com/containerd/containerd/blob/main/docs/getting-started.md#customizing-containerd) file with `containerd config default > /etc/containerd/config.toml`, in this config file edit the `"root"`-entry (default value is `/var/lib/containerd`). +4. Finally, you can now build the FastSurfer image with `python Docker/build.py ... --attest`. This will add the additional flags to the docker build command. diff --git a/Docker/build.py b/Docker/build.py index d86b4afd..136ef65d 100755 --- a/Docker/build.py +++ b/Docker/build.py @@ -210,12 +210,6 @@ def make_parser() -> argparse.ArgumentParser: metavar="image[:tag]", help="""tag build stage/target as [:]""", ) - parser.add_argument( - "--attest", - action="store_true", - help="add sbom and provenance attestation (requires docker-container buildkit " - "builder created with 'docker buildx create')", - ) parser.add_argument( "--target", default="runtime", @@ -270,6 +264,21 @@ def make_parser() -> argparse.ArgumentParser: expert = parser.add_argument_group('Expert options') + parser.add_argument( + "--attest", + action="store_true", + help="add sbom and provenance attestation (requires docker-container buildkit " + "builder created with 'docker buildx create')", + ) + parser.add_argument( + "--action", + choices=("load", "push"), + default="load", + help="Which action to perform after building the image: " + "'load' loads the image into the current docker context (default), " + "'push' pushes the image to the registry (needs --tag /" + ":", + ) expert.add_argument( "--freesurfer_build_image", type=docker_image, @@ -361,6 +370,7 @@ def docker_build_image( context: Path | str = ".", dry_run: bool = False, attestation: bool = False, + action: Literal["load", "push"] = "load", **kwargs) -> None: """ Build a docker image. @@ -383,6 +393,9 @@ def docker_build_image( argument to docker buildx build. attestation : bool, default=False Whether to create sbom and provenance attestation + action : "load", "push", default="load" + The operation to perform after the image is built (only in the attestation + pipeline, otherwise will load). Additional kwargs add additional build flags to the build command in the following manner: "_" is replaced by "-" in the keyword name and each sequence entry is passed @@ -400,6 +413,9 @@ def docker_build_image( if docker_cmd is None: raise FileNotFoundError("Could not locate the docker executable") + if action not in ("load", "push"): + raise ValueError(f"Invalid Value for 'action' {action}, must be load or push.") + def to_pair(key, values): if isinstance(values, Sequence) and isinstance(values, (str, bytes)): values = [values] @@ -445,7 +461,8 @@ def to_pair(key, values): ) if not can_use_default_builder: args.extend(["--builder", alternative_builder]) - args.extend(["--output", f"type=docker,name={image_name}", "--load"]) + image_type = "registry" if action == "push" else "docker" + args.extend(["--output", f"type={image_type},name={image_name}", "--" + action]) args.extend(("-t", image_name)) params = [to_pair(*a) for a in kwargs.items()] args.extend(["-f", str(dockerfile)] + list(chain(*params))) @@ -497,6 +514,8 @@ def main( raise ValueError(f"Invalid target: {target}") if device not in get_args(AllDeviceType): raise ValueError(f"Invalid device: {device}") + if keywords.get("action", "load") == "push": + kwargs["action"] = "push" # special case to add extra environment variables to better support AWS and ROCm if device.startswith("cu") and target == "runtime": target = "runtime_cuda"