diff --git a/jupyter-pytorch-full/rockcraft.yaml b/jupyter-pytorch-full/rockcraft.yaml index eb7ef2b..e378b02 100644 --- a/jupyter-pytorch-full/rockcraft.yaml +++ b/jupyter-pytorch-full/rockcraft.yaml @@ -7,9 +7,9 @@ description: | Both PyTorch and Jupyter are installed in Conda environment, which is automatically activated. -version: v1.8.0_20.04_1 # version format: __ +version: "1.8.0" license: Apache-2.0 -base: ubuntu:20.04 +base: ubuntu@20.04 run-user: _daemon_ services: jupyter: @@ -26,7 +26,7 @@ services: LANGUAGE: en_US.UTF-8 LC_ALL: en_US.UTF-8 PATH: /opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - PYTHONPATH: /opt/conda/lib/python3.8/site-packages/ + PYTHONPATH: /opt/conda/lib/python3.11/site-packages/ command: ./jupyter lab --notebook-dir="/home/jovyan" --ip=0.0.0.0 --no-browser --port=8888 --ServerApp.token="" --ServerApp.password="" --ServerApp.allow_origin="*" --ServerApp.base_url="/" --ServerApp.authenticate_prometheus="False" working-dir: /opt/conda/bin/ platforms: @@ -61,12 +61,13 @@ parts: - unzip - vim - wget + - xz-utils - zip kubectl: plugin: nil stage-snaps: - - kubectl/1.25/stable + - kubectl/1.27/stable organize: kubectl: bin/kubectl stage: @@ -97,14 +98,20 @@ parts: - wget build-environment: - MINIFORGE_ARCH: "x86_64" - - MINIFORGE_VERSION: "4.10.1-4" - - PIP_VERSION: "21.1.2" - - PYTHON_VERSION: "3.8.10" + - MINIFORGE_VERSION: "23.3.1-1" + - PIP_VERSION: "23.2.1" + - PYTHON_VERSION: "3.11.6" - HOME: "/home/jovyan" - CONDA_DIR: "/opt/conda" - CONDA_BIN: "/opt/conda/bin" + - JUPYTERLAB_VERSION: "3.6.6" + - JUPYTER_VERSION: "6.5.6" + - PYTORCH_VERSION: "2.1.0" + - TORCHAUDIO_VERSION: "2.1.0" + - TORCHVISION_VERSION: "0.16.0" + - MLFLOW_VERSION: "2.1.1" stage-packages: - - python3.8-distutils + - python3-distutils override-build: | set -xe mkdir -p "$CONDA_DIR" "$HOME" @@ -128,7 +135,7 @@ parts: "${CONDA_BIN}/conda" config --system --set show_channel_urls true echo "conda ${MINIFORGE_VERSION:0:-2}" >> "${CONDA_DIR}/conda-meta/pinned" - echo "python ${PYTHON_VERSION}" >> "${CONDA_DIR}/conda-meta/pinned" + echo "python ==${PYTHON_VERSION}" >> "${CONDA_DIR}/conda-meta/pinned" # Install the correct versions of python, conda, pip "${CONDA_BIN}/conda" install -y -q \ @@ -141,20 +148,47 @@ parts: "${CONDA_BIN}/conda" clean -a -f -y # Install the jupyter python requirements + echo "jupyterlab ==${JUPYTERLAB_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned + echo "notebook ==${JUPYTER_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned + "${CONDA_BIN}/conda" install -y -q \ + jupyterlab==${JUPYTERLAB_VERSION} \ + notebook==${JUPYTER_VERSION} cp ${CRAFT_PART_SRC}/components/example-notebook-servers/jupyter/requirements.txt jupyter-requirements.txt "${CONDA_BIN}/pip" install --no-cache-dir -r jupyter-requirements.txt # Generate a jupyter lab config + "${CONDA_BIN}/jupyter" notebook --generate-config "${CONDA_BIN}/jupyter" lab --generate-config + "${CONDA_BIN}/jupyter" labextension disable "@jupyterlab/apputils-extension:announcements" + + # Install the jupyter-pytorch requirements + "${CONDA_BIN}/pip" install --quiet --no-cache-dir --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \ + torch==${PYTORCH_VERSION} \ + torchvision==${TORCHVISION_VERSION} \ + torchaudio==${TORCHAUDIO_VERSION} + cp $CRAFT_PART_SRC/components/example-notebook-servers/jupyter-pytorch/requirements.txt requirements.txt + "${CONDA_BIN}/pip" install --no-cache-dir -r requirements.txt - # Install the jupyter-pytorch cpu requirements - cp $CRAFT_PART_SRC/components/example-notebook-servers/jupyter-pytorch/cpu-requirements.txt cpu-requirements.txt - "${CONDA_BIN}/pip" install --no-cache-dir -r cpu-requirements.txt - - # Install the jupyter-pytorch-full cpu requirements + # Install the jupyter-pytorch-full requirements + "${CONDA_BIN}/mamba" install -y -q \ + bokeh==3.2.2 \ + cloudpickle==2.2.1 \ + dill==0.3.7 \ + ipympl==0.9.3 \ + matplotlib==3.8.0 \ + pandas==2.1.1 \ + scikit-image==0.22.0 \ + scikit-learn==1.3.1 \ + scipy==1.11.3 \ + seaborn==0.13.0 \ + xgboost==1.7.6 cp $CRAFT_PART_SRC/components/example-notebook-servers/jupyter-pytorch-full/requirements.txt requirements.txt "${CONDA_BIN}/pip" install --no-cache-dir -r requirements.txt + # Install mlflow so we can use this with mlflow https://github.com/canonical/data-science-stack/issues/47#issuecomment-2004136344 + "${CONDA_BIN}/pip" install --quiet --no-cache-dir \ + mlflow==${MLFLOW_VERSION} + # Create some directories for staging files mkdir -p "${CRAFT_PART_INSTALL}/opt" "${CRAFT_PART_INSTALL}/etc" "${CRAFT_PART_INSTALL}/home" diff --git a/jupyter-pytorch-full/tests/imports.py b/jupyter-pytorch-full/tests/imports.py index 5a4f9e2..f380061 100755 --- a/jupyter-pytorch-full/tests/imports.py +++ b/jupyter-pytorch-full/tests/imports.py @@ -12,10 +12,8 @@ import torchaudio # kubeflow packages -# TO-DO verfiy proper kfp import. Upgrade might be needed. -#import kfp +import kfp import kfp_server_api -import kfserving # common packages import bokeh @@ -26,18 +24,14 @@ import jupyterlab_git import matplotlib import pandas -# TO-DO verify how exactly scikit-image is installed -# /opt/conda/lib/python3.8/site-packages/scikit_image-0.18.1.dist-info -#import scikit_image -# TO-DO verify how exactly scikit-learn is installed -# /opt/conda/lib/python3.8/site-packages/scikit_learn-0.24.2.dist-info -#import scikit_learn +import sklearn +import skimage import scipy import seaborn import xgboost -# pytorch packages -import fastai +# mlflow package +import mlflow # this string is expected by test script print("PASSED") diff --git a/jupyter-pytorch-full/tests/test_access.py b/jupyter-pytorch-full/tests/test_access.py index 7982157..d357c5b 100644 --- a/jupyter-pytorch-full/tests/test_access.py +++ b/jupyter-pytorch-full/tests/test_access.py @@ -3,15 +3,11 @@ # See LICENSE file for licensing details. # # -from pathlib import Path - -import os import subprocess -import time import requests import tenacity -import yaml +from charmed_kubeflow_chisme.rock import CheckRock @tenacity.retry( stop=tenacity.stop_after_attempt(5), @@ -24,11 +20,9 @@ def check_notebook_server_up(url): def main(): """Test running container and imports.""" - rock = yaml.safe_load(Path("rockcraft.yaml").read_text()) - name = rock["name"] - rock_version = rock["version"] - arch = list(rock["platforms"].keys())[0] - rock_image = f"{name}_{rock_version}_{arch}" + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" print(f"Running {LOCAL_ROCK_IMAGE}") diff --git a/jupyter-pytorch-full/tests/test_imports.py b/jupyter-pytorch-full/tests/test_imports.py old mode 100755 new mode 100644 index a46f9a5..4d293de --- a/jupyter-pytorch-full/tests/test_imports.py +++ b/jupyter-pytorch-full/tests/test_imports.py @@ -3,24 +3,21 @@ # See LICENSE file for licensing details. # # -from pathlib import Path - import os import subprocess -import yaml + +from charmed_kubeflow_chisme.rock import CheckRock def main(): """Test running container and imports.""" - rock = yaml.safe_load(Path("rockcraft.yaml").read_text()) - name = rock["name"] - rock_version = rock["version"] - arch = list(rock["platforms"].keys())[0] - rock_image = f"{name}_{rock_version}_{arch}" + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" print(f"Running command in {LOCAL_ROCK_IMAGE}") - script_command = ["bash", "-c", "export PYTHONPATH=$PYTHONPATH:/opt/conda/lib/python3.8/site-packages/keras/api/keras/wrappers/:/opt/conda/lib/python3.8/site-packages/ && python3 /home/jovyan/imports.py"] + script_command = ["bash", "-c", "export HOME=/home/jovyan && export PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && PYTHONPATH=/opt/conda/lib/python3.11/site-packages/ python3.11 /home/jovyan/imports.py"] pwd = os.getcwd() output = subprocess.run(["docker", "run", "-v", f"{pwd}/tests/:/home/jovyan", LOCAL_ROCK_IMAGE, "exec", "pebble", "exec"] + script_command, stdout=subprocess.PIPE).stdout.decode('utf-8') assert "PASSED" in output diff --git a/jupyter-pytorch-full/tests/test_rock.py b/jupyter-pytorch-full/tests/test_rock.py index 7c5bba5..b205274 100644 --- a/jupyter-pytorch-full/tests/test_rock.py +++ b/jupyter-pytorch-full/tests/test_rock.py @@ -2,25 +2,21 @@ # See LICENSE file for licensing details. # # -from pathlib import Path - import pytest import subprocess -import yaml -from pytest_operator.plugin import OpsTest + +from charmed_kubeflow_chisme.rock import CheckRock @pytest.mark.abort_on_fail -def test_rock(ops_test: OpsTest): +def test_rock(): """Test rock.""" - rock = yaml.safe_load(Path("rockcraft.yaml").read_text()) - name = rock["name"] - rock_version = rock["version"] - arch = list(rock["platforms"].keys())[0] - rock_image = f"{name}_{rock_version}_{arch}" + check_rock = CheckRock("rockcraft.yaml") + rock_image = check_rock.get_name() + rock_version = check_rock.get_version() + rock_services = check_rock.get_services() LOCAL_ROCK_IMAGE = f"{rock_image}:{rock_version}" # verify ROCK service - rock_services = rock["services"] assert rock_services["jupyter"] assert rock_services["jupyter"]["startup"] == "enabled" diff --git a/jupyter-pytorch-full/tox.ini b/jupyter-pytorch-full/tox.ini index 3f72c52..a8f7617 100644 --- a/jupyter-pytorch-full/tox.ini +++ b/jupyter-pytorch-full/tox.ini @@ -3,6 +3,7 @@ [tox] skipsdist = True skip_missing_interpreters = True +envlist = pack, export-to-docker, sanity, integration [testenv] setenv = @@ -12,29 +13,47 @@ setenv = CHARM_BRANCH=main LOCAL_CHARM_DIR=charm_repo -[testenv:unit] +[testenv:pack] passenv = * allowlist_externals = - bash - tox rockcraft -deps = - charmed-kubeflow-chisme - juju<4.0 - pytest - pytest-operator - ops - tenacity commands = - # build and pack rock rockcraft pack + +[testenv:export-to-docker] +passenv = * +allowlist_externals = + bash + skopeo + yq +commands = + # export already packed rock to docker bash -c 'NAME=$(yq eval .name rockcraft.yaml) && \ VERSION=$(yq eval .version rockcraft.yaml) && \ - ARCH=$(yq eval -r ".platforms | keys" rockcraft.yaml | cut -d" " -f2) && \ - ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}" && \ - sudo skopeo --insecure-policy copy oci-archive:$ROCK.rock docker-daemon:$ROCK:$VERSION && \ - docker save $ROCK > $ROCK.tar' + ARCH=$(yq eval ".platforms | keys | .[0]" rockcraft.yaml) && \ + ROCK="$\{NAME\}_$\{VERSION\}_$\{ARCH\}.rock" && \ + DOCKER_IMAGE=$NAME:$VERSION && \ + echo "Exporting $ROCK to docker as $DOCKER_IMAGE" && \ + skopeo --insecure-policy copy oci-archive:$ROCK docker-daemon:$DOCKER_IMAGE' + +[testenv:sanity] +passenv = * +deps = + pytest + charmed-kubeflow-chisme + PyYaml + requests + tenacity +commands = # run rock tests pytest -v --tb native --show-capture=all --log-cli-level=INFO {posargs} {toxinidir}/tests python {toxinidir}/tests/test_imports.py python {toxinidir}/tests/test_access.py + +[testenv:integration] +passenv = * +allowlist_externals = + echo +commands = + # TODO: Implement integration tests here + echo "WARNING: This is a placeholder test - no test is implemented here."