Fix pytorch cpu jupyter server rock (#79)

* Fix pytorch cpu jupyter server rock * Fix tox in jupyter pytorch cpu
canonical · Mar 21, 2024 · 3058193 · 3058193
1 parent 0719441
commit 3058193
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 70 deletions.
diff --git a/jupyter-pytorch-full/rockcraft.yaml b/jupyter-pytorch-full/rockcraft.yaml
@@ -7,9 +7,9 @@ description: |
 
   Both PyTorch and Jupyter are installed in Conda environment, which is
   automatically activated.
-version: v1.8.0_20.04_1 # version format: <KF-upstream-version>_<base-version>_<Charmed-KF-version>
+version: "1.8.0"
 license: Apache-2.0
-base: ubuntu:20.04
+base: ubuntu@20.04
 run-user: _daemon_
 services:
   jupyter:
@@ -26,7 +26,7 @@ services:
        LANGUAGE: en_US.UTF-8
        LC_ALL: en_US.UTF-8
        PATH: /opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-       PYTHONPATH: /opt/conda/lib/python3.8/site-packages/
+       PYTHONPATH: /opt/conda/lib/python3.11/site-packages/
     command: ./jupyter lab --notebook-dir="/home/jovyan" --ip=0.0.0.0 --no-browser --port=8888 --ServerApp.token="" --ServerApp.password="" --ServerApp.allow_origin="*" --ServerApp.base_url="/" --ServerApp.authenticate_prometheus="False"
     working-dir: /opt/conda/bin/
 platforms:
@@ -61,12 +61,13 @@ parts:
       - unzip
       - vim
       - wget
+      - xz-utils
       - zip
 
   kubectl:
     plugin: nil
     stage-snaps:
-      - kubectl/1.25/stable
+      - kubectl/1.27/stable
     organize:
       kubectl: bin/kubectl
     stage:
@@ -97,14 +98,20 @@ parts:
       - wget
     build-environment:
       - MINIFORGE_ARCH: "x86_64"
-      - MINIFORGE_VERSION: "4.10.1-4"
-      - PIP_VERSION: "21.1.2"
-      - PYTHON_VERSION: "3.8.10"
+      - MINIFORGE_VERSION: "23.3.1-1"
+      - PIP_VERSION: "23.2.1"
+      - PYTHON_VERSION: "3.11.6"
       - HOME: "/home/jovyan"
       - CONDA_DIR: "/opt/conda"
       - CONDA_BIN: "/opt/conda/bin"
+      - JUPYTERLAB_VERSION: "3.6.6"
+      - JUPYTER_VERSION: "6.5.6"
+      - PYTORCH_VERSION: "2.1.0"
+      - TORCHAUDIO_VERSION: "2.1.0"
+      - TORCHVISION_VERSION: "0.16.0"
+      - MLFLOW_VERSION: "2.1.1"
     stage-packages:
-      - python3.8-distutils
+      - python3-distutils
     override-build: |
       set -xe
       mkdir -p "$CONDA_DIR" "$HOME"
@@ -128,7 +135,7 @@ parts:
       "${CONDA_BIN}/conda" config --system --set show_channel_urls true
 
       echo "conda ${MINIFORGE_VERSION:0:-2}" >> "${CONDA_DIR}/conda-meta/pinned"
-      echo "python ${PYTHON_VERSION}" >> "${CONDA_DIR}/conda-meta/pinned"
+      echo "python ==${PYTHON_VERSION}" >> "${CONDA_DIR}/conda-meta/pinned"
 
       # Install the correct versions of python, conda, pip
       "${CONDA_BIN}/conda" install -y -q \
@@ -141,20 +148,47 @@ parts:
       "${CONDA_BIN}/conda" clean -a -f -y
 
       # Install the jupyter python requirements
+      echo "jupyterlab ==${JUPYTERLAB_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned
+      echo "notebook ==${JUPYTER_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned
+      "${CONDA_BIN}/conda" install -y -q \
+        jupyterlab==${JUPYTERLAB_VERSION} \
+        notebook==${JUPYTER_VERSION}
       cp ${CRAFT_PART_SRC}/components/example-notebook-servers/jupyter/requirements.txt jupyter-requirements.txt
       "${CONDA_BIN}/pip" install --no-cache-dir -r jupyter-requirements.txt
 
       # Generate a jupyter lab config
+      "${CONDA_BIN}/jupyter" notebook --generate-config
       "${CONDA_BIN}/jupyter" lab --generate-config
+      "${CONDA_BIN}/jupyter" labextension disable "@jupyterlab/apputils-extension:announcements"
+
+      # Install the jupyter-pytorch requirements
+      "${CONDA_BIN}/pip" install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
+        torch==${PYTORCH_VERSION} \
+        torchvision==${TORCHVISION_VERSION} \
+        torchaudio==${TORCHAUDIO_VERSION}
+      cp $CRAFT_PART_SRC/components/example-notebook-servers/jupyter-pytorch/requirements.txt requirements.txt
+      "${CONDA_BIN}/pip" install --no-cache-dir -r requirements.txt
 
-      # Install the jupyter-pytorch cpu requirements
-      cp $CRAFT_PART_SRC/components/example-notebook-servers/jupyter-pytorch/cpu-requirements.txt cpu-requirements.txt
-      "${CONDA_BIN}/pip" install --no-cache-dir -r cpu-requirements.txt
-
-      # Install the jupyter-pytorch-full cpu requirements
+      # Install the jupyter-pytorch-full requirements
+      "${CONDA_BIN}/mamba" install -y -q \
+        bokeh==3.2.2 \
+        cloudpickle==2.2.1 \
+        dill==0.3.7 \
+        ipympl==0.9.3 \
+        matplotlib==3.8.0 \
+        pandas==2.1.1 \
+        scikit-image==0.22.0 \
+        scikit-learn==1.3.1 \
+        scipy==1.11.3 \
+        seaborn==0.13.0 \
+        xgboost==1.7.6
       cp $CRAFT_PART_SRC/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt requirements.txt
       "${CONDA_BIN}/pip" install --no-cache-dir -r requirements.txt
 
+      # Install mlflow so we can use this with mlflow https://github.com/canonical/data-science-stack/issues/47#issuecomment-2004136344
+      "${CONDA_BIN}/pip" install --quiet --no-cache-dir \
+        mlflow==${MLFLOW_VERSION}
+
       # Create some directories for staging files
       mkdir -p "${CRAFT_PART_INSTALL}/opt" "${CRAFT_PART_INSTALL}/etc" "${CRAFT_PART_INSTALL}/home"
 

diff --git a/jupyter-pytorch-full/tests/imports.py b/jupyter-pytorch-full/tests/imports.py
@@ -12,10 +12,8 @@
 import torchaudio
 
 # kubeflow packages
-# TO-DO verfiy proper kfp import. Upgrade might be needed.
-#import kfp
+import kfp
 import kfp_server_api
-import kfserving
 
 # common packages
 import bokeh
@@ -26,18 +24,14 @@
 import jupyterlab_git
 import matplotlib
 import pandas
-# TO-DO verify how exactly scikit-image is installed
-# /opt/conda/lib/python3.8/site-packages/scikit_image-0.18.1.dist-info
-#import scikit_image
-# TO-DO verify how exactly scikit-learn is installed
-# /opt/conda/lib/python3.8/site-packages/scikit_learn-0.24.2.dist-info
-#import scikit_learn
+import sklearn
+import skimage 
 import scipy
 import seaborn
 import xgboost
 
-# pytorch packages
-import fastai
+# mlflow package
+import mlflow
 
 # this string is expected by test script
 print("PASSED")
diff --git a/jupyter-pytorch-full/tests/test_access.py b/jupyter-pytorch-full/tests/test_access.py
@@ -3,15 +3,11 @@
 # See LICENSE file for licensing details.
 #
 #
-from pathlib import Path
-
-import os
 import subprocess
-import time
 import requests
 import tenacity
-import yaml
 
+from charmed_kubeflow_chisme.rock import CheckRock
 
 @tenacity.retry(
 stop=tenacity.stop_after_attempt(5),
@@ -24,11 +20,9 @@ def check_notebook_server_up(url):
 
 def main():
     """Test running container and imports."""
-    rock = yaml.safe_load(Path("rockcraft.yaml").read_text())
-    name = rock["name"]
-    rock_version = rock["version"]
-    arch = list(rock["platforms"].keys())[0]
-    rock_image = f"{name}_{rock_version}_{arch}"
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
     LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
 
     print(f"Running {LOCAL_ROCK_IMAGE}")

diff --git a/jupyter-pytorch-full/tests/test_imports.py b/jupyter-pytorch-full/tests/test_imports.py
@@ -3,24 +3,21 @@
 # See LICENSE file for licensing details.
 #
 #
-from pathlib import Path
-
 import os
 import subprocess
-import yaml
+
+from charmed_kubeflow_chisme.rock import CheckRock
 
 
 def main():
     """Test running container and imports."""
-    rock = yaml.safe_load(Path("rockcraft.yaml").read_text())
-    name = rock["name"]
-    rock_version = rock["version"]
-    arch = list(rock["platforms"].keys())[0]
-    rock_image = f"{name}_{rock_version}_{arch}"
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
     LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
 
     print(f"Running command in {LOCAL_ROCK_IMAGE}")
-    script_command = ["bash", "-c", "export PYTHONPATH=$PYTHONPATH:/opt/conda/lib/python3.8/site-packages/keras/api/keras/wrappers/:/opt/conda/lib/python3.8/site-packages/ && python3 /home/jovyan/imports.py"]
+    script_command = ["bash", "-c", "export HOME=/home/jovyan && export PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && PYTHONPATH=/opt/conda/lib/python3.11/site-packages/  python3.11 /home/jovyan/imports.py"]
     pwd = os.getcwd()
     output = subprocess.run(["docker", "run", "-v", f"{pwd}/tests/:/home/jovyan", LOCAL_ROCK_IMAGE, "exec", "pebble", "exec"] + script_command, stdout=subprocess.PIPE).stdout.decode('utf-8')
     assert "PASSED" in output

diff --git a/jupyter-pytorch-full/tests/test_rock.py b/jupyter-pytorch-full/tests/test_rock.py
@@ -2,25 +2,21 @@
 # See LICENSE file for licensing details.
 #
 #
-from pathlib import Path
-
 import pytest
 import subprocess
-import yaml
-from pytest_operator.plugin import OpsTest
+
+from charmed_kubeflow_chisme.rock import CheckRock
 
 @pytest.mark.abort_on_fail
-def test_rock(ops_test: OpsTest):
+def test_rock():
     """Test rock."""
-    rock = yaml.safe_load(Path("rockcraft.yaml").read_text())
-    name = rock["name"]
-    rock_version = rock["version"]
-    arch = list(rock["platforms"].keys())[0]
-    rock_image = f"{name}_{rock_version}_{arch}"
+    check_rock = CheckRock("rockcraft.yaml")
+    rock_image = check_rock.get_name()
+    rock_version = check_rock.get_version()
+    rock_services = check_rock.get_services()
     LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}"
 
     # verify ROCK service
-    rock_services = rock["services"]
     assert rock_services["jupyter"]
     assert rock_services["jupyter"]["startup"] == "enabled"
 

diff --git a/jupyter-pytorch-full/tox.ini b/jupyter-pytorch-full/tox.ini
@@ -3,6 +3,7 @@
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
+envlist = pack, export-to-docker, sanity, integration
 
 [testenv]
 setenv =
@@ -12,29 +13,47 @@ setenv =
     CHARM_BRANCH=main
     LOCAL_CHARM_DIR=charm_repo
 
-[testenv:unit]
+[testenv:pack]
 passenv = *
 allowlist_externals =
-    bash
-    tox
     rockcraft
-deps =
-    charmed-kubeflow-chisme
-    juju<4.0
-    pytest
-    pytest-operator
-    ops
-    tenacity
 commands =
-    # build and pack rock
     rockcraft pack
+
+[testenv:export-to-docker]
+passenv = *
+allowlist_externals =
+    bash
+    skopeo
+    yq
+commands =
+    # export already packed rock to docker
     bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \
              VERSION=$(yq eval .version rockcraft.yaml) && \
-             ARCH=$(yq eval -r ".platforms | keys" rockcraft.yaml | cut -d" " -f2) && \
-             ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \
-             sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \
-             docker save $ROCK > $ROCK.tar'
+             ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \
+             ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \
+             DOCKER_IMAGE=$NAME:$VERSION && \
+             echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \
+             skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE'
+
+[testenv:sanity]
+passenv = *
+deps =
+    pytest
+    charmed-kubeflow-chisme
+    PyYaml
+    requests
+    tenacity
+commands =
     # run rock tests
     pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests
     python {toxinidir}/tests/test_imports.py
     python {toxinidir}/tests/test_access.py
+
+[testenv:integration]
+passenv = *
+allowlist_externals =
+    echo
+commands =
+    # TODO: Implement integration tests here
+    echo "WARNING: This is a placeholder test - no test is implemented here."