From 7e05536932460550b449d294c790dfc9a7f92158 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 1 Oct 2024 22:31:13 +0000 Subject: [PATCH 1/6] wip: simplify devcontainer to fewer layers --- .devcontainer/Dockerfile | 26 +++++++++++++++++-- .../scripts/common/prep-deps-files.sh | 5 ++++ conda-envs/mlos.yml | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8702d314cb..faf009bf60 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -FROM mcr.microsoft.com/devcontainers/miniconda:3 AS base +FROM debian:stable-slim AS base # Add some additional packages for the devcontainer terminal environment. USER root @@ -11,17 +11,39 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ less colordiff \ curl jq \ ripgrep \ - vim-nox neovim python3-pynvim \ + vim-nox neovim \ make \ rename \ + sudo \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && echo "C-w: unix-filename-rubout" >> /etc/inputrc # Also tweak C-w to stop at slashes as well instead of just spaces +RUN useradd -m -U -G sudo vscode + +RUN echo "Setup miniconda" \ + && curl -Ss https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc > /etc/apt/trusted.gpg.d/conda.asc \ + && gpg --keyring /etc/apt/trusted.gpg.d/conda.asc --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 \ + && echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/conda.asc] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list \ + && apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends conda \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && adduser vscode conda + # Set some cache dirs to be owned by the vscode user even as we're currently # executing as root to build the container image. # NOTE: We do *not* mark these as volumes - it doesn't help rebuilding at all. +ARG VSCODE_UID=1000 +ARG VSCODE_GID=1000 +ARG DOCKER_GID=999 + +RUN groupmod --non-unique --gid $VSCODE_GID vscode \ + && usermod --non-unique --uid $VSCODE_UID --gid $NODE_GID vscode \ + && chown -R $NODE_UID:$NODE_GID /home/vscode \ + && groupadd --non-unique --gid $DOCKER_GID docker \ + && adduser vscode docker + ARG PIP_CACHE_DIR=/var/cache/pip ENV PIP_CACHE_DIR=/var/cache/pip RUN mkdir -p ${PIP_CACHE_DIR} \ diff --git a/.devcontainer/scripts/common/prep-deps-files.sh b/.devcontainer/scripts/common/prep-deps-files.sh index 46d8eb46a8..f2c5373932 100755 --- a/.devcontainer/scripts/common/prep-deps-files.sh +++ b/.devcontainer/scripts/common/prep-deps-files.sh @@ -17,6 +17,11 @@ set -x scriptdir=$(dirname "$(readlink -f "$0")") cd "$scriptdir" +if ! type -p python3; then + sudo -n apt-get update + sudo -n apt-get -y install --no-install-recommends python3 +fi + cat /tmp/conda-tmp/mlos.yml \ | sed 's|#.*||' \ | egrep -v -e '--editable' -e '^\s*$' \ diff --git a/conda-envs/mlos.yml b/conda-envs/mlos.yml index 34d1db429d..4f5f12c8d6 100644 --- a/conda-envs/mlos.yml +++ b/conda-envs/mlos.yml @@ -20,7 +20,7 @@ dependencies: - pandas - pyarrow - swig - - libpq + - libpq<17.0 - python - pip: - bump2version From 5b682bde5f0974a67c021a9ec6b05ad08d013c8f Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 1 Oct 2024 22:55:35 +0000 Subject: [PATCH 2/6] wip --- .devcontainer/Dockerfile | 43 ++++++++++--------- .../scripts/common/prep-deps-files.sh | 5 --- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index faf009bf60..25ed2cc5a6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,7 +9,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends \ bash bash-completion \ less colordiff \ - curl jq \ + curl gpg \ + jq \ ripgrep \ vim-nox neovim \ make \ @@ -22,13 +23,26 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ RUN useradd -m -U -G sudo vscode RUN echo "Setup miniconda" \ - && curl -Ss https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc > /etc/apt/trusted.gpg.d/conda.asc \ - && gpg --keyring /etc/apt/trusted.gpg.d/conda.asc --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 \ - && echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/conda.asc] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list \ + && curl -Ss https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/conda.gpg \ + && gpg --keyring /etc/apt/trusted.gpg.d/conda.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 \ + && echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/conda.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list \ && apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y --no-install-recommends conda \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ - && adduser vscode conda + && adduser vscode conda \ + && /opt/conda/bin/conda init --system bash \ + && echo "# Upgrade conda and use strict priorities" \ + && echo "# Use the mamba solver (necessary for some quality of life speedups due to required packages to support Windows)" \ + && umask 0002 \ + && /opt/conda/bin/conda config --set channel_priority strict \ + && /opt/conda/bin/conda info \ + && /opt/conda/bin/conda update -v -y -n base -c defaults --all \ + && /opt/conda/bin/conda list -n base \ + && /opt/conda/bin/conda install -v -y -n base conda-libmamba-solver \ + && /opt/conda/bin/conda config --set solver libmamba \ + && /opt/conda/bin/conda list -n base \ + && /opt/conda/bin/conda clean -v -y -a \ + && /opt/conda/bin/conda run -n base pip cache purge # Set some cache dirs to be owned by the vscode user even as we're currently # executing as root to build the container image. @@ -58,20 +72,6 @@ RUN mkdir -p ${CONDA_PKGS_DIRS} \ USER vscode:conda -# Upgrade conda and use strict priorities -# Use the mamba solver (necessary for some quality of life speedups due to -# required packages to support Windows) -RUN umask 0002 \ - && /opt/conda/bin/conda config --set channel_priority strict \ - && /opt/conda/bin/conda info \ - && /opt/conda/bin/conda update -v -y -n base -c defaults --all \ - && /opt/conda/bin/conda list -n base \ - && /opt/conda/bin/conda install -v -y -n base conda-libmamba-solver \ - && /opt/conda/bin/conda config --set solver libmamba \ - && /opt/conda/bin/conda list -n base \ - && /opt/conda/bin/conda clean -v -y -a \ - && /opt/conda/bin/conda run -n base pip cache purge - # No longer relevant since we're using conda-forge in the environment files by default now. ## Update the base. This helps save space by making sure the same version ## python is used for both the base env and mlos env. @@ -95,7 +95,10 @@ RUN /opt/conda/bin/conda init bash \ && /opt/conda/bin/conda config --set solver libmamba # Prepare the mlos_deps.yml file in a cross platform way. -FROM mcr.microsoft.com/devcontainers/miniconda:3 AS deps-prep +FROM debian:stable-slim AS deps-prep +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends \ + python3-minimal python3-setuptools COPY --chown=vscode:conda . /tmp/conda-tmp/ RUN /tmp/conda-tmp/prep-deps-files.sh \ && ls -l /tmp/conda-tmp/ # && cat /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/mlos_deps.yml diff --git a/.devcontainer/scripts/common/prep-deps-files.sh b/.devcontainer/scripts/common/prep-deps-files.sh index f2c5373932..46d8eb46a8 100755 --- a/.devcontainer/scripts/common/prep-deps-files.sh +++ b/.devcontainer/scripts/common/prep-deps-files.sh @@ -17,11 +17,6 @@ set -x scriptdir=$(dirname "$(readlink -f "$0")") cd "$scriptdir" -if ! type -p python3; then - sudo -n apt-get update - sudo -n apt-get -y install --no-install-recommends python3 -fi - cat /tmp/conda-tmp/mlos.yml \ | sed 's|#.*||' \ | egrep -v -e '--editable' -e '^\s*$' \ From fc532270d962d87573d87bd862a4de1834a5b0d7 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Wed, 2 Oct 2024 19:44:07 +0000 Subject: [PATCH 3/6] Refactor to reduce space --- .devcontainer/Dockerfile | 137 +++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 79 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 25ed2cc5a6..77c20fe34b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -FROM debian:stable-slim AS base +FROM mcr.microsoft.com/vscode/devcontainers/base AS base # Add some additional packages for the devcontainer terminal environment. USER root @@ -9,10 +9,10 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends \ bash bash-completion \ less colordiff \ - curl gpg \ + curl gpg ca-certificates \ jq \ ripgrep \ - vim-nox neovim \ + vim-nox neovim python3-pynvim \ make \ rename \ sudo \ @@ -20,43 +20,23 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && echo "C-w: unix-filename-rubout" >> /etc/inputrc # Also tweak C-w to stop at slashes as well instead of just spaces -RUN useradd -m -U -G sudo vscode +# Prepare the mlos_deps.yml file in a cross platform way. +FROM mcr.microsoft.com/vscode/devcontainers/base AS deps-prep +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends \ + python3-minimal python3-setuptools +COPY --chown=vscode . /tmp/conda-tmp/ +RUN /tmp/conda-tmp/prep-deps-files.sh \ + && ls -l /tmp/conda-tmp/ # && cat /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/mlos_deps.yml -RUN echo "Setup miniconda" \ - && curl -Ss https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/conda.gpg \ - && gpg --keyring /etc/apt/trusted.gpg.d/conda.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 \ - && echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/conda.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list \ - && apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends conda \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ - && adduser vscode conda \ - && /opt/conda/bin/conda init --system bash \ - && echo "# Upgrade conda and use strict priorities" \ - && echo "# Use the mamba solver (necessary for some quality of life speedups due to required packages to support Windows)" \ - && umask 0002 \ - && /opt/conda/bin/conda config --set channel_priority strict \ - && /opt/conda/bin/conda info \ - && /opt/conda/bin/conda update -v -y -n base -c defaults --all \ - && /opt/conda/bin/conda list -n base \ - && /opt/conda/bin/conda install -v -y -n base conda-libmamba-solver \ - && /opt/conda/bin/conda config --set solver libmamba \ - && /opt/conda/bin/conda list -n base \ - && /opt/conda/bin/conda clean -v -y -a \ - && /opt/conda/bin/conda run -n base pip cache purge +FROM base AS conda # Set some cache dirs to be owned by the vscode user even as we're currently # executing as root to build the container image. # NOTE: We do *not* mark these as volumes - it doesn't help rebuilding at all. -ARG VSCODE_UID=1000 -ARG VSCODE_GID=1000 -ARG DOCKER_GID=999 - -RUN groupmod --non-unique --gid $VSCODE_GID vscode \ - && usermod --non-unique --uid $VSCODE_UID --gid $NODE_GID vscode \ - && chown -R $NODE_UID:$NODE_GID /home/vscode \ - && groupadd --non-unique --gid $DOCKER_GID docker \ - && adduser vscode docker +RUN addgroup conda \ + && adduser vscode conda ARG PIP_CACHE_DIR=/var/cache/pip ENV PIP_CACHE_DIR=/var/cache/pip @@ -72,55 +52,54 @@ RUN mkdir -p ${CONDA_PKGS_DIRS} \ USER vscode:conda -# No longer relevant since we're using conda-forge in the environment files by default now. -## Update the base. This helps save space by making sure the same version -## python is used for both the base env and mlos env. -#RUN umask 0002 \ -# && /opt/conda/bin/conda update -v -y -n base -c defaults --all \ -# && /opt/conda/bin/conda update -v -y -n base -c defaults conda python \ -# && /opt/conda/bin/conda clean -v -y -a \ -# && /opt/conda/bin/conda run -n base pip cache purge - -# Install some additional editor packages for the base environment. -RUN umask 0002 \ - && /opt/conda/bin/conda run -n base pip install --no-cache-dir -U pynvim - -# Setup (part of) the mlos environment in the devcontainer. -# NOTEs: -# - The mlos_deps.yml file is prepared by the prep-container-build script(s). -# - The rest happens during first container start once the source is available. -# See Also: updateContentCommand in .devcontainer/devcontainer.json -RUN mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ -RUN /opt/conda/bin/conda init bash \ - && /opt/conda/bin/conda config --set solver libmamba - -# Prepare the mlos_deps.yml file in a cross platform way. -FROM debian:stable-slim AS deps-prep -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends \ - python3-minimal python3-setuptools -COPY --chown=vscode:conda . /tmp/conda-tmp/ -RUN /tmp/conda-tmp/prep-deps-files.sh \ - && ls -l /tmp/conda-tmp/ # && cat /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/mlos_deps.yml +# Try and prime the devcontainer's ssh known_hosts keys with the github one for scripted calls. +RUN mkdir -p /home/vscode/.ssh \ + && ( \ + grep -q ^github.com /home/vscode/.ssh/known_hosts \ + || ssh-keyscan github.com | tee -a /home/vscode/.ssh/known_hosts \ + ) -# Install some additional dependencies for the mlos environment. -# Make sure they have conda group ownership to make the devcontainer more -# reliable useable across vscode uid changes. -FROM base AS devcontainer -USER vscode COPY --from=deps-prep --chown=vscode:conda /tmp/conda-tmp/mlos_deps.yml /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/ -RUN umask 0002 \ + +# Combine the installation of miniconda and the mlos dependencies into a single step in order to save space. +# This allows the mlos env to reference the base env's packages without duplication across layers. +RUN echo "Setup miniconda" \ + && curl -Ss https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/conda.gpg > /dev/null \ + && gpg --keyring /etc/apt/trusted.gpg.d/conda.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 \ + && echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/conda.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list \ + && sudo apt-get update \ + && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends conda \ + && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* \ + && echo "# Adjust the conda installation to be user/group writable." \ + && sudo /opt/conda/bin/conda init --system \ + && sudo chgrp -R conda /opt/conda \ + && sudo chmod -R g+wX /opt/conda \ + && find /opt/conda -type d -print0 | xargs -0 sudo chmod -c g+s \ + && umask 0002 \ + && echo "# Use conda-forge first to get the latest versions of packages " \ + && echo "# and reduce duplication with mlos env (which also uses conda-forge first)." \ + && echo "# Upgrade conda and use strict priorities" \ + && echo "# Use the mamba solver (necessary for some quality of life speedups due to required packages to support Windows)" \ + && /opt/conda/bin/conda init \ + && /opt/conda/bin/conda config --set channel_priority strict \ + && /opt/conda/bin/conda info \ + && /opt/conda/bin/conda update -v -y -n base -c conda-forge -c defaults --all \ + && /opt/conda/bin/conda list -n base \ + && /opt/conda/bin/conda install -v -y -n base -c conda-forge -c defaults conda-libmamba-solver \ + && /opt/conda/bin/conda config --system --set solver libmamba \ + && echo "# Install some additional editor packages for the base environment." \ + && /opt/conda/bin/conda run -n base pip install --no-cache-dir -U pynvim \ + && echo "# Clean up conda cache to save some space." \ + && /opt/conda/bin/conda list -n base \ + && /opt/conda/bin/conda clean -v -y -a \ + && /opt/conda/bin/conda run -n base pip cache purge \ + && echo "# Install some additional dependencies for the mlos environment." \ + && echo "# Make sure they have conda group ownership to make the devcontainer more" \ + && echo "# reliable useable across vscode uid changes." \ && sg conda -c "/opt/conda/bin/conda env create -n mlos -v -f /tmp/conda-tmp/mlos_deps.yml" \ && sg conda -c "/opt/conda/bin/conda run -n mlos pip install --no-cache-dir -U -r /tmp/conda-tmp/combined.requirements.txt" \ && sg conda -c "/opt/conda/bin/conda run -n mlos pip cache purge" \ && sg conda -c "/opt/conda/bin/conda clean -v -y -a" \ - && mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ -RUN mkdir -p /home/vscode/.conda/envs \ + && mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ \ + && mkdir -p /home/vscode/.conda/envs \ && ln -s /opt/conda/envs/mlos /home/vscode/.conda/envs/mlos - -# Try and prime the devcontainer's ssh known_hosts keys with the github one for scripted calls. -RUN mkdir -p /home/vscode/.ssh \ - && ( \ - grep -q ^github.com /home/vscode/.ssh/known_hosts \ - || ssh-keyscan github.com | tee -a /home/vscode/.ssh/known_hosts \ - ) From ecd90f4c19ccf3f7619e980a0ac949c99a617eb9 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 3 Oct 2024 22:20:03 +0000 Subject: [PATCH 4/6] Tweaks --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 8eb06a70f8..5f4f14a4ab 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +export PATH := $(PATH):/opt/conda/bin + CONDA_ENV_NAME ?= mlos PYTHON_VERSION := $(shell echo "${CONDA_ENV_NAME}" | sed -r -e 's/^mlos[-]?//') ENV_YML := conda-envs/${CONDA_ENV_NAME}.yml From 0eee4ab411892db8ec1ec84f1d7e5aeecd823024 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 3 Oct 2024 22:48:41 +0000 Subject: [PATCH 5/6] include the conda path in container --- .devcontainer/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 77c20fe34b..99b21ede54 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -103,3 +103,6 @@ RUN echo "Setup miniconda" \ && mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ \ && mkdir -p /home/vscode/.conda/envs \ && ln -s /opt/conda/envs/mlos /home/vscode/.conda/envs/mlos + +#ENV PATH=/opt/conda/bin:$PATH +ENV PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin From 36c8f7867060b79de3564ebfd5a5a6798cd7d688 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 3 Oct 2024 22:48:50 +0000 Subject: [PATCH 6/6] revert --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 5f4f14a4ab..8eb06a70f8 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -export PATH := $(PATH):/opt/conda/bin - CONDA_ENV_NAME ?= mlos PYTHON_VERSION := $(shell echo "${CONDA_ENV_NAME}" | sed -r -e 's/^mlos[-]?//') ENV_YML := conda-envs/${CONDA_ENV_NAME}.yml