Make torch available

Makefile

@@ -85,7 +85,6 @@ generate-dockerfiles: clean jupyterlab rstudio remote-desktop sas docker-stacks-
 pytorch tensorflow: .output
 	$(CAT) \
 		$(SRC)/0_cpu.Dockerfile \
-		$(SRC)/1_CUDA-$($(@)-CUDA).Dockerfile \
 		$(SRC)/2_$@.Dockerfile \
 	> $(TMP)/$@.Dockerfile
 

docker-bits/2_cpu.Dockerfile

@@ -1,17 +1,9 @@
 # Create conda environment (CPU only) with many useful packages.
-
-RUN conda create -n pycpu --yes \
-      python==3.11.0 ipython==8.11.0 sphinx==6.1.3 \
-      boto==2.49.0 s3fs==2023.3.0 \
-      dos2unix==7.4.1 parallel==20230122 \
-      dask==2023.3.0 numpy==1.24.2 pandas==1.5.3 pyarrow==11.0.0 scipy==1.10.1 \
-      scikit-learn==1.2.2 xgboost==1.7.1 \
-      matplotlib==3.7.1 pillow==9.4.0 \
-      gdal==3.6.2 geopandas==0.12.2 rasterio==1.3.6 \
-      opencv==4.7.0 scikit-image==0.19.3 \
-      gensim==4.3.0 nltk==3.8.1 spacy==3.5.0 \
-      pytorch==1.13.1 torchaudio==0.13.1 torchvision==0.14.1 cpuonly==2.0 \
-      -c pytorch -c conda-forge && \
-    conda clean --all -f -y && \
+RUN mamba install pytorch \
+        torchvision \
+        torchaudio \
+        cpuonly \
+        -c pytorch && \
+    mamba clean --all -f -y && \
     fix-permissions $CONDA_DIR && \
     fix-permissions /home/$NB_USER

docker-bits/2_pytorch.Dockerfile

@@ -1,11 +1,19 @@
-#Install PyTorch
-RUN conda create -n torch python=3.9 && \
-   conda install -n torch --quiet --yes -c pytorch \
-     'pytorch==1.13.1' \
-     'torchvision==0.14.1' \
-     'ipykernel==6.21.3' \
-     'torchtext==0.14.1' \
-   && \
-   conda clean --all -f -y && \
-   fix-permissions $CONDA_DIR && \
-   fix-permissions /home/$NB_USER
+# Install PyTorch GPU Packages and enable PyTorch IPyKernel
+RUN mamba create -n torch python=3.11 && \
+    mamba install -n torch --quiet --yes -c pytorch -c nvidia \
+        ipykernel \
+        pytorch \
+        torchvision \
+        torchaudio \
+        # gputil has nvidia-smi
+        gputil \
+        # pytorch-cuda are the nvidia cuda drivers
+        pytorch-cuda=11.8 \
+    && \
+    conda clean --all -f -y && \
+    fix-permissions $CONDA_DIR && \
+    fix-permissions /home/$NB_USER && \
+    source activate torch && \
+    python -m ipykernel install --user --name torch --display-name "PyTorch"
+
+

output/jupyterlab-pytorch/Dockerfile

@@ -31,135 +31,28 @@ RUN pip install --force-reinstall cryptography==39.0.1 && \
    fix-permissions /home/$NB_USER
 
 ###############################
-###  docker-bits/1_CUDA-11.8.0.Dockerfile
+###  docker-bits/2_pytorch.Dockerfile
 ###############################
 
-# Cuda stuff for v11.8.0
-
-## https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/base/Dockerfile
-
-###########################
-### Base
-###########################
-
-ENV NVARCH x86_64
-
-ENV NVIDIA_REQUIRE_CUDA "cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511 brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511 brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516 brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516 brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516"
-ENV NV_CUDA_CUDART_VERSION 11.8.89-1
-ENV NV_CUDA_COMPAT_PACKAGE cuda-compat-11-8
-ENV OS_VER ubuntu2204
-
-ARG TARGETARCH
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gnupg2 curl ca-certificates && \
-    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/${OS_VER}/${NVARCH}/3bf863cc.pub | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/nvidia.gpg && \
-    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/${OS_VER}/${NVARCH} /" > /etc/apt/sources.list.d/cuda.list && \
-    apt-get purge --autoremove -y curl \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV CUDA_VERSION 11.8.0
-
-# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    cuda-cudart-11-8=${NV_CUDA_CUDART_VERSION} \
-    ${NV_CUDA_COMPAT_PACKAGE} \
-    && rm -rf /var/lib/apt/lists/*
-
-# Required for nvidia-docker v1
-RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
-    && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
-
-ENV CUDA_DIR "/usr/local/cuda"
-ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
-ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:$CUDA_DIR/lib64
-ENV XLA_FLAGS "--xla_gpu_cuda_data_dir=$CUDA_DIR"
-
-# nvidia-container-runtime
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-
-# ###########################
-# ### Devel
-# ###########################
-# # https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/devel/Dockerfile
-#
-# $(curl -s https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/devel/Dockerfile)
-
-###########################
-### Runtime
-###########################
-# https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/runtime/Dockerfile
-
-ENV NV_CUDA_LIB_VERSION 11.8.0-1
-
-ENV NV_NVTX_VERSION 11.8.86-1
-ENV NV_LIBNPP_VERSION 11.8.0.86-1
-ENV NV_LIBNPP_PACKAGE libnpp-11-8=${NV_LIBNPP_VERSION}
-ENV NV_LIBCUSPARSE_VERSION 11.7.5.86-1
-
-ENV NV_LIBCUBLAS_PACKAGE_NAME libcublas-11-8
-ENV NV_LIBCUBLAS_VERSION 11.11.3.6-1
-ENV NV_LIBCUBLAS_PACKAGE ${NV_LIBCUBLAS_PACKAGE_NAME}=${NV_LIBCUBLAS_VERSION}
-
-ENV NV_LIBNCCL_PACKAGE_NAME libnccl2
-ENV NV_LIBNCCL_PACKAGE_VERSION 2.15.5-1
-ENV NCCL_VERSION 2.15.5-1
-ENV NV_LIBNCCL_PACKAGE ${NV_LIBNCCL_PACKAGE_NAME}=${NV_LIBNCCL_PACKAGE_VERSION}+cuda11.8
-
-ARG TARGETARCH
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    cuda-libraries-11-8=${NV_CUDA_LIB_VERSION} \
-    cuda-toolkit-11-8 \
-    ${NV_LIBNPP_PACKAGE} \
-    cuda-nvtx-11-8=${NV_NVTX_VERSION} \
-    libcusparse-11-8=${NV_LIBCUSPARSE_VERSION} \
-    ${NV_LIBCUBLAS_PACKAGE} \
-    ${NV_LIBNCCL_PACKAGE} \
-    && rm -rf /var/lib/apt/lists/*
-
-# Keep apt from auto upgrading the cublas and nccl packages. See https://gitlab.com/nvidia/container-images/cuda/-/issues/88
-RUN apt-mark hold ${NV_LIBCUBLAS_PACKAGE_NAME} ${NV_LIBNCCL_PACKAGE_NAME}
-
-# Add entrypoint items
-ENV NVIDIA_PRODUCT_NAME="CUDA"
-
-###########################
-### CudNN
-###########################
-# https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/runtime/cudnn8/Dockerfile
-
-ENV NV_CUDNN_VERSION 8.6.0.163
-ENV NV_CUDNN_PACKAGE_NAME "libcudnn8"
-
-ENV NV_CUDNN_PACKAGE "libcudnn8=$NV_CUDNN_VERSION-1+cuda11.8"
-
-ARG TARGETARCH
-
-LABEL com.nvidia.cudnn.version="${NV_CUDNN_VERSION}"
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    ${NV_CUDNN_PACKAGE} \
-    && apt-mark hold ${NV_CUDNN_PACKAGE_NAME} \
-    && rm -rf /var/lib/apt/lists/*
-
+# Install PyTorch GPU Packages and enable PyTorch IPyKernel
+RUN mamba create -n torch python=3.11 && \
+    mamba install -n torch --quiet --yes -c pytorch -c nvidia \
+        ipykernel \
+        pytorch \
+        torchvision \
+        torchaudio \
+        # gputil has nvidia-smi
+        gputil \
+        # pytorch-cuda are the nvidia cuda drivers
+        pytorch-cuda=11.8 \
+    && \
+    conda clean --all -f -y && \
+    fix-permissions $CONDA_DIR && \
+    fix-permissions /home/$NB_USER && \
+    source activate torch && \
+    python -m ipykernel install --user --name torch --display-name "PyTorch"
 
-###############################
-###  docker-bits/2_pytorch.Dockerfile
-###############################
 
-#Install PyTorch
-RUN conda create -n torch python=3.9 && \
-   conda install -n torch --quiet --yes -c pytorch \
-     'pytorch==1.13.1' \
-     'torchvision==0.14.1' \
-     'ipykernel==6.21.3' \
-     'torchtext==0.14.1' \
-   && \
-   conda clean --all -f -y && \
-   fix-permissions $CONDA_DIR && \
-   fix-permissions /home/$NB_USER
 
 ###############################
 ###  docker-bits/3_Kubeflow.Dockerfile

output/jupyterlab-tensorflow/Dockerfile

@@ -30,121 +30,6 @@ RUN pip install --force-reinstall cryptography==39.0.1 && \
    fix-permissions $CONDA_DIR && \
    fix-permissions /home/$NB_USER
 
-###############################
-###  docker-bits/1_CUDA-11.8.0.Dockerfile
-###############################
-
-# Cuda stuff for v11.8.0
-
-## https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/base/Dockerfile
-
-###########################
-### Base
-###########################
-
-ENV NVARCH x86_64
-
-ENV NVIDIA_REQUIRE_CUDA "cuda>=11.8 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511 brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511 brand=titanrtx,driver>=510,driver<511 brand=tesla,driver>=515,driver<516 brand=unknown,driver>=515,driver<516 brand=nvidia,driver>=515,driver<516 brand=nvidiartx,driver>=515,driver<516 brand=geforce,driver>=515,driver<516 brand=geforcertx,driver>=515,driver<516 brand=quadro,driver>=515,driver<516 brand=quadrortx,driver>=515,driver<516 brand=titan,driver>=515,driver<516 brand=titanrtx,driver>=515,driver<516"
-ENV NV_CUDA_CUDART_VERSION 11.8.89-1
-ENV NV_CUDA_COMPAT_PACKAGE cuda-compat-11-8
-ENV OS_VER ubuntu2204
-
-ARG TARGETARCH
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gnupg2 curl ca-certificates && \
-    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/${OS_VER}/${NVARCH}/3bf863cc.pub | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/nvidia.gpg && \
-    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/${OS_VER}/${NVARCH} /" > /etc/apt/sources.list.d/cuda.list && \
-    apt-get purge --autoremove -y curl \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV CUDA_VERSION 11.8.0
-
-# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    cuda-cudart-11-8=${NV_CUDA_CUDART_VERSION} \
-    ${NV_CUDA_COMPAT_PACKAGE} \
-    && rm -rf /var/lib/apt/lists/*
-
-# Required for nvidia-docker v1
-RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
-    && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
-
-ENV CUDA_DIR "/usr/local/cuda"
-ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
-ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:$CUDA_DIR/lib64
-ENV XLA_FLAGS "--xla_gpu_cuda_data_dir=$CUDA_DIR"
-
-# nvidia-container-runtime
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-
-# ###########################
-# ### Devel
-# ###########################
-# # https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/devel/Dockerfile
-#
-# $(curl -s https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/devel/Dockerfile)
-
-###########################
-### Runtime
-###########################
-# https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/runtime/Dockerfile
-
-ENV NV_CUDA_LIB_VERSION 11.8.0-1
-
-ENV NV_NVTX_VERSION 11.8.86-1
-ENV NV_LIBNPP_VERSION 11.8.0.86-1
-ENV NV_LIBNPP_PACKAGE libnpp-11-8=${NV_LIBNPP_VERSION}
-ENV NV_LIBCUSPARSE_VERSION 11.7.5.86-1
-
-ENV NV_LIBCUBLAS_PACKAGE_NAME libcublas-11-8
-ENV NV_LIBCUBLAS_VERSION 11.11.3.6-1
-ENV NV_LIBCUBLAS_PACKAGE ${NV_LIBCUBLAS_PACKAGE_NAME}=${NV_LIBCUBLAS_VERSION}
-
-ENV NV_LIBNCCL_PACKAGE_NAME libnccl2
-ENV NV_LIBNCCL_PACKAGE_VERSION 2.15.5-1
-ENV NCCL_VERSION 2.15.5-1
-ENV NV_LIBNCCL_PACKAGE ${NV_LIBNCCL_PACKAGE_NAME}=${NV_LIBNCCL_PACKAGE_VERSION}+cuda11.8
-
-ARG TARGETARCH
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    cuda-libraries-11-8=${NV_CUDA_LIB_VERSION} \
-    cuda-toolkit-11-8 \
-    ${NV_LIBNPP_PACKAGE} \
-    cuda-nvtx-11-8=${NV_NVTX_VERSION} \
-    libcusparse-11-8=${NV_LIBCUSPARSE_VERSION} \
-    ${NV_LIBCUBLAS_PACKAGE} \
-    ${NV_LIBNCCL_PACKAGE} \
-    && rm -rf /var/lib/apt/lists/*
-
-# Keep apt from auto upgrading the cublas and nccl packages. See https://gitlab.com/nvidia/container-images/cuda/-/issues/88
-RUN apt-mark hold ${NV_LIBCUBLAS_PACKAGE_NAME} ${NV_LIBNCCL_PACKAGE_NAME}
-
-# Add entrypoint items
-ENV NVIDIA_PRODUCT_NAME="CUDA"
-
-###########################
-### CudNN
-###########################
-# https://gitlab.com/nvidia/container-images/cuda/-/raw/ee72a6fef178d135e8366e5c88e15df39ff83c21/dist/11.8.0/ubuntu1804/runtime/cudnn8/Dockerfile
-
-ENV NV_CUDNN_VERSION 8.6.0.163
-ENV NV_CUDNN_PACKAGE_NAME "libcudnn8"
-
-ENV NV_CUDNN_PACKAGE "libcudnn8=$NV_CUDNN_VERSION-1+cuda11.8"
-
-ARG TARGETARCH
-
-LABEL com.nvidia.cudnn.version="${NV_CUDNN_VERSION}"
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    ${NV_CUDNN_PACKAGE} \
-    && apt-mark hold ${NV_CUDNN_PACKAGE_NAME} \
-    && rm -rf /var/lib/apt/lists/*
-
-
 ###############################
 ###  docker-bits/2_tensorflow.Dockerfile
 ###############################