diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8702d314cb..99b21ede54 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -FROM mcr.microsoft.com/devcontainers/miniconda:3 AS base +FROM mcr.microsoft.com/vscode/devcontainers/base AS base # Add some additional packages for the devcontainer terminal environment. USER root @@ -9,19 +9,35 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends \ bash bash-completion \ less colordiff \ - curl jq \ + curl gpg ca-certificates \ + jq \ ripgrep \ vim-nox neovim python3-pynvim \ make \ rename \ + sudo \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && echo "C-w: unix-filename-rubout" >> /etc/inputrc # Also tweak C-w to stop at slashes as well instead of just spaces +# Prepare the mlos_deps.yml file in a cross platform way. +FROM mcr.microsoft.com/vscode/devcontainers/base AS deps-prep +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends \ + python3-minimal python3-setuptools +COPY --chown=vscode . /tmp/conda-tmp/ +RUN /tmp/conda-tmp/prep-deps-files.sh \ + && ls -l /tmp/conda-tmp/ # && cat /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/mlos_deps.yml + +FROM base AS conda + # Set some cache dirs to be owned by the vscode user even as we're currently # executing as root to build the container image. # NOTE: We do *not* mark these as volumes - it doesn't help rebuilding at all. +RUN addgroup conda \ + && adduser vscode conda + ARG PIP_CACHE_DIR=/var/cache/pip ENV PIP_CACHE_DIR=/var/cache/pip RUN mkdir -p ${PIP_CACHE_DIR} \ @@ -36,66 +52,57 @@ RUN mkdir -p ${CONDA_PKGS_DIRS} \ USER vscode:conda -# Upgrade conda and use strict priorities -# Use the mamba solver (necessary for some quality of life speedups due to -# required packages to support Windows) -RUN umask 0002 \ +# Try and prime the devcontainer's ssh known_hosts keys with the github one for scripted calls. +RUN mkdir -p /home/vscode/.ssh \ + && ( \ + grep -q ^github.com /home/vscode/.ssh/known_hosts \ + || ssh-keyscan github.com | tee -a /home/vscode/.ssh/known_hosts \ + ) + +COPY --from=deps-prep --chown=vscode:conda /tmp/conda-tmp/mlos_deps.yml /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/ + +# Combine the installation of miniconda and the mlos dependencies into a single step in order to save space. +# This allows the mlos env to reference the base env's packages without duplication across layers. +RUN echo "Setup miniconda" \ + && curl -Ss https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/conda.gpg > /dev/null \ + && gpg --keyring /etc/apt/trusted.gpg.d/conda.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 \ + && echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/conda.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | sudo tee /etc/apt/sources.list.d/conda.list \ + && sudo apt-get update \ + && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends conda \ + && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* \ + && echo "# Adjust the conda installation to be user/group writable." \ + && sudo /opt/conda/bin/conda init --system \ + && sudo chgrp -R conda /opt/conda \ + && sudo chmod -R g+wX /opt/conda \ + && find /opt/conda -type d -print0 | xargs -0 sudo chmod -c g+s \ + && umask 0002 \ + && echo "# Use conda-forge first to get the latest versions of packages " \ + && echo "# and reduce duplication with mlos env (which also uses conda-forge first)." \ + && echo "# Upgrade conda and use strict priorities" \ + && echo "# Use the mamba solver (necessary for some quality of life speedups due to required packages to support Windows)" \ + && /opt/conda/bin/conda init \ && /opt/conda/bin/conda config --set channel_priority strict \ && /opt/conda/bin/conda info \ - && /opt/conda/bin/conda update -v -y -n base -c defaults --all \ + && /opt/conda/bin/conda update -v -y -n base -c conda-forge -c defaults --all \ && /opt/conda/bin/conda list -n base \ - && /opt/conda/bin/conda install -v -y -n base conda-libmamba-solver \ - && /opt/conda/bin/conda config --set solver libmamba \ + && /opt/conda/bin/conda install -v -y -n base -c conda-forge -c defaults conda-libmamba-solver \ + && /opt/conda/bin/conda config --system --set solver libmamba \ + && echo "# Install some additional editor packages for the base environment." \ + && /opt/conda/bin/conda run -n base pip install --no-cache-dir -U pynvim \ + && echo "# Clean up conda cache to save some space." \ && /opt/conda/bin/conda list -n base \ && /opt/conda/bin/conda clean -v -y -a \ - && /opt/conda/bin/conda run -n base pip cache purge - -# No longer relevant since we're using conda-forge in the environment files by default now. -## Update the base. This helps save space by making sure the same version -## python is used for both the base env and mlos env. -#RUN umask 0002 \ -# && /opt/conda/bin/conda update -v -y -n base -c defaults --all \ -# && /opt/conda/bin/conda update -v -y -n base -c defaults conda python \ -# && /opt/conda/bin/conda clean -v -y -a \ -# && /opt/conda/bin/conda run -n base pip cache purge - -# Install some additional editor packages for the base environment. -RUN umask 0002 \ - && /opt/conda/bin/conda run -n base pip install --no-cache-dir -U pynvim - -# Setup (part of) the mlos environment in the devcontainer. -# NOTEs: -# - The mlos_deps.yml file is prepared by the prep-container-build script(s). -# - The rest happens during first container start once the source is available. -# See Also: updateContentCommand in .devcontainer/devcontainer.json -RUN mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ -RUN /opt/conda/bin/conda init bash \ - && /opt/conda/bin/conda config --set solver libmamba - -# Prepare the mlos_deps.yml file in a cross platform way. -FROM mcr.microsoft.com/devcontainers/miniconda:3 AS deps-prep -COPY --chown=vscode:conda . /tmp/conda-tmp/ -RUN /tmp/conda-tmp/prep-deps-files.sh \ - && ls -l /tmp/conda-tmp/ # && cat /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/mlos_deps.yml - -# Install some additional dependencies for the mlos environment. -# Make sure they have conda group ownership to make the devcontainer more -# reliable useable across vscode uid changes. -FROM base AS devcontainer -USER vscode -COPY --from=deps-prep --chown=vscode:conda /tmp/conda-tmp/mlos_deps.yml /tmp/conda-tmp/combined.requirements.txt /tmp/conda-tmp/ -RUN umask 0002 \ + && /opt/conda/bin/conda run -n base pip cache purge \ + && echo "# Install some additional dependencies for the mlos environment." \ + && echo "# Make sure they have conda group ownership to make the devcontainer more" \ + && echo "# reliable useable across vscode uid changes." \ && sg conda -c "/opt/conda/bin/conda env create -n mlos -v -f /tmp/conda-tmp/mlos_deps.yml" \ && sg conda -c "/opt/conda/bin/conda run -n mlos pip install --no-cache-dir -U -r /tmp/conda-tmp/combined.requirements.txt" \ && sg conda -c "/opt/conda/bin/conda run -n mlos pip cache purge" \ && sg conda -c "/opt/conda/bin/conda clean -v -y -a" \ - && mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ -RUN mkdir -p /home/vscode/.conda/envs \ + && mkdir -p /opt/conda/pkgs/cache/ && chown -R vscode:conda /opt/conda/pkgs/cache/ \ + && mkdir -p /home/vscode/.conda/envs \ && ln -s /opt/conda/envs/mlos /home/vscode/.conda/envs/mlos -# Try and prime the devcontainer's ssh known_hosts keys with the github one for scripted calls. -RUN mkdir -p /home/vscode/.ssh \ - && ( \ - grep -q ^github.com /home/vscode/.ssh/known_hosts \ - || ssh-keyscan github.com | tee -a /home/vscode/.ssh/known_hosts \ - ) +#ENV PATH=/opt/conda/bin:$PATH +ENV PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin