Update kernels (#11)

* add jpy pkgs * fix jupyter kernels * add full-stack
QPod · Jun 14, 2024 · 149776e · 149776e
1 parent ea03af1
commit 149776e
Show file tree

Hide file tree

Showing 10 changed files with 106 additions and 54 deletions.
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -73,9 +73,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: |
-          source ./tool.sh
-          build_image openresty latest docker_openresty/Dockerfile
-          push_image
+          source ./tool.sh && build_image openresty latest docker_openresty/Dockerfile && push_image
 
   ## DevBox - base
   qpod_base-dev:
@@ -85,24 +83,40 @@ jobs:
       - uses: actions/checkout@v4
       - run: |
           source ./tool.sh && free_diskspace
-          build_image base-dev latest docker_dev_box/Dockerfile \
+          build_image base-dev latest docker_dev_box/dev.Dockerfile \
             --build-arg "BASE_IMG=node" \
             --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
             --build-arg "ARG_PROFILE_VSCODE=base"
             alias_image base-dev latest developer latest && push_image dev
-  ## DevBox - full stack
+
+  ## DevBox - data science stack
   qpod_data-science-dev:
     name: 'data-science-dev'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: |
           source ./tool.sh && free_diskspace
-          build_image data-science-dev latest docker_dev_box/Dockerfile \
+          build_image data-science-dev latest docker_dev_box/dev.Dockerfile \
             --build-arg "BASE_IMG=data-science-stack" \
             --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
             --build-arg "ARG_PROFILE_VSCODE=base"
           push_image
+
+  ## DevBox - full stack
+  qpod_full-stack-dev:
+    name: 'full-stack-dev'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          source ./tool.sh && free_diskspace
+          build_image full-stack-dev latest docker_dev_box/dev.Dockerfile \
+            --build-arg "BASE_IMG=full-stack" \
+            --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
+            --build-arg "ARG_PROFILE_VSCODE=base"
+          push_image
+
   ## DevBox - cuda
   qpod_cuda-dev:
     name: 'full-cuda,cuda-dev'
@@ -111,15 +125,16 @@ jobs:
       - uses: actions/checkout@v4
       - run: |
           source ./tool.sh && free_diskspace
-          build_image cuda-dev latest docker_dev_box/Dockerfile \
+          build_image cuda-dev latest docker_dev_box/dev.Dockerfile \
             --build-arg "BASE_IMG=core-cuda" \
             --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
             --build-arg "ARG_PROFILE_VSCODE=base"
           alias_image cuda-dev latest full-cuda latest && push_image
 
+
   ## Sync all images in this build (listed by "names") to mirror registry.
   sync_images:
-    needs: ["qpod_cuda-dev"]
+    needs: ["qpod_cuda-dev", "qpod_data-science-dev", "qpod_full-stack-dev"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

diff --git a/docker_app_clash/Dockerfile b/docker_app_clash/Dockerfile
@@ -2,7 +2,8 @@ FROM alpine AS builder
 
 # https://github.com/haishanh/yacd
 WORKDIR /tmp
-RUN wget https://github.com/haishanh/yacd/archive/gh-pages.zip \
+RUN set -eux \
+ && wget https://github.com/haishanh/yacd/archive/gh-pages.zip \
  && unzip ./*.zip && rm -rf *.zip \
  && mv ./* ./ui
 

diff --git a/docker_casdoor/Dockerfile b/docker_casdoor/Dockerfile
@@ -6,14 +6,16 @@ ARG BASE_IMG="node"
 
 FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG_BUILD} as builder
 COPY work /opt/utils/
-RUN source /opt/utils/script-setup-casdoor.sh \
+RUN set -ex \
+ && source /opt/utils/script-setup-casdoor.sh \
  && setup_casdoor
 
 
 FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
 COPY --from=builder /opt/casdoor /opt/casdoor
 COPY work/app.conf /opt/casdoor/conf/app.conf
-RUN mkdir -pv /root/web && ln -sf /opt/casdoor/web/build /root/web/
+RUN set -ex \
+ && mkdir -pv /root/web && ln -sf /opt/casdoor/web/build /root/web/
 
 LABEL maintainer="haobibo@gmail.com"
 ENV RUNNING_IN_DOCKER true

diff --git a/docker_dev_box/README.md b/docker_dev_box/README.md
@@ -2,7 +2,7 @@
 
 ## Develop and Debug - Single User
 
-```bash
+```shell
 IMG="qpod/developer"
 # IMG="registry.cn-hangzhou.aliyuncs.com/qpod/full-stack-dev"
 
@@ -16,3 +16,16 @@ docker run -d --restart=always \
 
 sleep 5s && docker logs QPod-lab-dev 2>&1|grep token=
 ```
+
+Debug building:
+
+```shell
+IMG="qpod/rust"
+docker run --rm -it \
+    --name=QPod-lab-dev --hostname=QPod \
+    -p 18888-18890:8888-8890 \
+    -v $(pwd):/root/ -w /root/ \
+    $IMG bash
+
+docker exec -it QPod-lab-dev bash
+```
diff --git a/docker_dev_box/Dockerfile → docker_dev_box/dev.Dockerfile b/docker_dev_box/Dockerfile → docker_dev_box/dev.Dockerfile
@@ -16,22 +16,24 @@ ARG ARG_KEEP_NODEJS=true
 
 COPY work /opt/utils/
 
-# Setup Jupyter: Basic Configurations and Extensions
-RUN mkdir -pv /opt/conda/etc/jupyter/ \
+RUN set -ex && source /opt/utils/script-utils.sh \
+ # Setup Jupyter: Basic Configurations and Extensions
+ && mkdir -pv /opt/conda/etc/jupyter/ \
  && mv /opt/utils/etc_jupyter/* /opt/conda/etc/jupyter/ && rm -rf /opt/utils/etc_jupyter \
  && mv /opt/utils/start-*.sh /usr/local/bin/ && chmod +x /usr/local/bin/start-*.sh \
  && source /opt/utils/script-devbox-jupyter.sh \
- && for profile in $(echo $ARG_PROFILE_JUPYTER | tr "," "\n") ; do ( setup_jupyter_${profile} || true ) ; done
-
-# If not keeping NodeJS, remove NoedJS to reduce image size
-RUN ${ARG_KEEP_NODEJS:-true}  || ( echo "Removing Node/NPM..." && rm -rf /usr/bin/node /usr/bin/npm /usr/bin/npx /opt/node )
-
-# If installing coder-server  # https://github.com/cdr/code-server/releases
-RUN source /opt/utils/script-devbox-vscode.sh \
- && for profile in $(echo $ARG_PROFILE_VSCODE | tr "," "\n") ; do ( setup_vscode_${profile} || true ) ; done
-
-# Clean up and display components version information...
-RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages
+ && for profile in $(echo $ARG_PROFILE_JUPYTER | tr "," "\n") ; do ( setup_jupyter_${profile} || true ) ; done \
+ # If installing coder-server  # https://github.com/cdr/code-server/releases
+ && source /opt/utils/script-devbox-vscode.sh \
+ && for profile in $(echo $ARG_PROFILE_VSCODE | tr "," "\n") ; do ( setup_vscode_${profile} || true ) ; done \
+ # If not keeping NodeJS, remove NoedJS to reduce image size
+ && if [ ${ARG_KEEP_NODEJS} = "false" ] ; then \
+      echo "Removing Node/NPM..." && rm -rf /usr/bin/node /usr/bin/npm /usr/bin/npx /opt/node ; \
+    else \
+      echo "Keep NodeJS as ARG_KEEP_NODEJS defiend as: ${ARG_KEEP_NODEJS}" ; \
+ fi \
+ # Clean up and display components version information...
+ && install__clean && list_installed_packages
 
 ENTRYPOINT ["tini", "-g", "--"]
 

diff --git a/docker_dev_box/hub.Dockerfile b/docker_dev_box/hub.Dockerfile
@@ -13,15 +13,18 @@ ARG ARG_KEEP_NODEJS=true
 
 COPY work /opt/utils/
 
-# Setup JupyterHub
-RUN source /opt/utils/script-devbox-jupyter.sh \
+RUN set -ex \
+ # Setup JupyterHub
+ && source /opt/utils/script-devbox-jupyter.sh \
  && mv /opt/utils/start-*.sh /usr/local/bin/ && chmod +x /usr/local/bin/start-*.sh \
  && for profile in $(echo $ARG_PROFILE_JUPYTER | tr "," "\n") ; do ( setup_jupyter_${profile} || true ) ; done \
  # If not keeping NodeJS, remove NoedJS to reduce image size, and install Traefik instead
- && ${ARG_KEEP_NODEJS:-true} || ( \
-       echo "Removing Node/NPM..." && rm -rf /usr/bin/node /usr/bin/npm /usr/bin/npx /opt/node \
-    && echo "Installing Traefik to server as proxy:" && source /opt/utils/script-setup.sh && setup_traefik \
- ) \
+ && if [ ${ARG_KEEP_NODEJS} = "false" ] ; then \
+      echo "Removing Node/NPM..." && rm -rf /usr/bin/node /usr/bin/npm /usr/bin/npx /opt/node ; \
+      echo "Installing Traefik to server as proxy:" && source /opt/utils/script-setup.sh && setup_traefik ; \
+    else \
+      echo "Keep NodeJS as ARG_KEEP_NODEJS defiend as: ${ARG_KEEP_NODEJS}" ; \
+ fi \
  # Clean up and display components version information...
  && source /opt/utils/script-utils.sh && install__clean && list_installed_packages
 

diff --git a/docker_dev_box/work/install_list_JPY_extend.pip b/docker_dev_box/work/install_list_JPY_extend.pip
@@ -2,9 +2,12 @@
 % Use percent char as line comment separator.
 
 ipyparallel                             % https://github.com/ipython/ipyparallel
+jupytext                                % https://github.com/mwouts/jupytext
 ipynb                                   % https://github.com/ipython/ipynb
-jupyter-resource-usage                  % https://github.com/jupyter-server/jupyter-resource-usage
 jupyterlab_server[openapi]              % https://github.com/jupyterlab/jupyterlab_server
+jupyter-resource-usage                  % https://github.com/jupyter-server/jupyter-resource-usage
+jupyterlab_rise                         % https://github.com/jupyterlab-contrib/rise
 jupyterlab-git                          % https://github.com/jupyterlab/jupyterlab-git
+jupyterlab-language-pack-zh-CN          % language pack: https://github.com/jupyterlab/language-packs/tree/main/language-packs/jupyterlab-language-pack-zh-CN
 % jupyter-collaboration                 % https://github.com/jupyterlab/jupyter-collaboration
 % jupyterlab-latex                      % bug on pypi version: https://github.com/jupyterlab/jupyterlab-latex
diff --git a/docker_dev_box/work/script-devbox-jupyter.sh b/docker_dev_box/work/script-devbox-jupyter.sh
@@ -2,16 +2,17 @@ source /opt/utils/script-utils.sh
 
 
 setup_jupyter_base() {
-     pip install -Uq --pre jupyterhub jupyterlab notebook ipywidgets jupyter-server-proxy \
+     pip install -Uq --pre jupyterlab notebook ipywidgets jupyter-server-proxy \
   && echo "@ Version of Jupyter Server: $(jupyter server --version)" \
   && echo "@ Version of Jupyter Lab: $(jupyter lab --version)" \
-  && echo "@ Version of Jupyter Notebook: $(jupyter notebook --version)" \
-  && echo "@ Version of JupyterHub: $(jupyterhub --version)"
+  && echo "@ Version of Jupyter Notebook: $(jupyter notebook --version)"
+
+  jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
 }
 
 
 setup_jupyter_kernels() {
-  echo "@ Jupyter Kernels RefList: https://github.com/jupyter/jupyter/wiki/Jupyter-kernels"
+     echo "@ Jupyter Kernels RefList: https://github.com/jupyter/jupyter/wiki/Jupyter-kernels"
 
      echo "@ Install Jupyter Kernel for Bash" \
   && pip install -Uq bash_kernel && python -m bash_kernel.install --sys-prefix
@@ -26,6 +27,7 @@ setup_jupyter_kernels() {
   # && /opt/node/bin/ijsinstall --install=global --spec-path=full \
   # && mv /usr/local/share/jupyter/kernels/javascript /opt/conda/share/jupyter/kernels/
 
+  ## alternative: https://github.com/jupyter-xeus/xeus-r
   ## checked @ 2024-0307  # TODO: help func requries proxy
      which R \
   && echo "@ Install Jupyter Kernel for R: https://github.com/melff/RKernel" \
@@ -43,25 +45,34 @@ setup_jupyter_kernels() {
   && $GOPATH/bin/gonb --install \
   && mv ~/.local/share/jupyter/kernels/gonb /opt/conda/share/jupyter/kernels/
 
-  ## TODO: buggy, cargo install sccache
+  ## checked @ 2024-0614
+  # alternative approaches to install evxcr_jupyter:
+  # && cargo install --locked evcxr_jupyter \
      which rustc \
-  && echo "@ Install Jupyter Kernel for Rust: https://github.com/evcxr/evcxr/issues/140" \
-  && cargo install --locked evcxr_jupyter \
-  && evcxr_jupyter --install --sys-prefix \
+  && echo "@ Install Jupyter Kernel for Rust: https://github.com/evcxr/evcxr/blob/main/evcxr_jupyter/README.md" \
+  && VER_EVCXR=$(curl -sL https://github.com/evcxr/evcxr/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[\d.]+' ) \
+  && URL_EVCXR="https://github.com/evcxr/evcxr/releases/download/v${VER_EVCXR}/evcxr_jupyter-v${VER_EVCXR}-x86_64-unknown-linux-gnu.tar.gz" \
+  && echo "Downloading evcxr version ${VER_EVCXR} from: ${URL_EVCXR}" \
+  && install_tar_gz $URL_EVCXR && mv /opt/evcxr* /tmp/evcxr && mv /tmp/evcxr/evcxr_jupyter /opt/cargo/bin/ \
+  && /opt/cargo/bin/evcxr_jupyter --install --sys-prefix \
   && mv ~/.local/share/jupyter/kernels/rust /opt/conda/share/jupyter/kernels/
 
-  ## TODO: buggy
-     which java \
-  && pip install -Uq pandas py4j
-  #&& pip install beakerx && beakerx install \
-  #&& jupyter labextension install beakerx-jupyterlab
-
   ## checked @ 2024-0307
      which julia \
   && echo "@ Install Jupyter Kernel for Julia: https://github.com/JuliaLang/IJulia.jl" \
   && julia -e "using Pkg; Pkg.add(\"IJulia\"); Pkg.precompile();" \
   && mv ~/.local/share/jupyter/kernels/julia* /opt/conda/share/jupyter/kernels/
 
+  ## Checked @ 2024-0614
+     which java \
+  && export JBANG_DIR=/opt/jbang && export PATH=${PATH}:${JBANG_DIR}/bin \
+  && echo "export JBANG_DIR=${JBANG_DIR}"         > /etc/profile.d/path-jbang.sh \
+  && echo 'export PATH=${PATH}:${JBANG_DIR}/bin' >> /etc/profile.d/path-jbang.sh \
+  && curl -Ls https://sh.jbang.dev | bash -s - app setup \
+  && ${JBANG_DIR}/bin/jbang trust add https://github.com/jupyter-java \
+  && ${JBANG_DIR}/bin/jbang install-kernel@jupyter-java \
+  && mv ~/.local/share/jupyter/kernels/jbang-* /opt/conda/share/jupyter/kernels/
+
   ## https://github.com/jupyter-xeus/xeus-octave  # TODO: to check
      which octave \
   && export PATH=/opt/octave/bin:$PATH \
@@ -75,18 +86,19 @@ setup_jupyter_extensions() {
      install_apt /opt/utils/install_list_JPY_extend.apt \
   && install_pip /opt/utils/install_list_JPY_extend.pip
 
-     echo "@ Jupyter Server Extension list: " && jupyter server extension list \
-  && echo "@ Jupyter Lab Extension list: " && jupyter labextension list \
+     echo "@ Jupyter Server Extension list: "   && jupyter server extension list \
+  && echo "@ Jupyter Lab Extension list: "      && jupyter labextension list \
   && echo "@ Jupyter Notebook Extension list: " && jupyter notebook extension list
 }
 
 
-setup_jupyter_hub() {
+setup_jupyter_hub() {  
    # ref1: https://github.com/jupyterhub/jupyterhub
    # ref2: https://github.com/jupyterhub/jupyterhub/blob/main/Dockerfile
-      which npm && npm install -g npm configurable-http-proxy
+      which npm && ( npm install -g npm configurable-http-proxy ) || ( echo "NPM not found!" && return 255 )
 
-      pip install -Uq oauthenticator jupyterhub-ldapauthenticator jupyterhub-kerberosauthenticator \
+      pip install -Uq --pre jupyterhub \
+   && pip install -Uq oauthenticator jupyterhub-ldapauthenticator jupyterhub-kerberosauthenticator \
    && pip install -Uq dockerspawner jupyterhub-kubespawner jupyterhub-systemdspawner wrapspawner \
    && pip install -Uq psutil pycurl jupyter_client jupyterhub \
    && pip install -Uq jupyterhub-traefik-proxy \

diff --git a/docker_keycloak/Dockerfile b/docker_keycloak/Dockerfile
@@ -8,8 +8,8 @@ LABEL maintainer="haobibo@gmail.com"
 
 COPY work /opt/utils/
 
-RUN source /opt/utils/script-setup-keycloak.sh \
- && setup_keycloak \
+RUN set -ex \
+ && source /opt/utils/script-setup-keycloak.sh && setup_keycloak \
  && echo "keycloak:x:0:root" >> /etc/group \
  && echo "keycloak:x:1000:0:keycloak user:/opt/keycloak:/sbin/nologin" >> /etc/passwd \
  && chown -R 1000:0 /opt/keycloak

diff --git a/docker_openresty/Dockerfile b/docker_openresty/Dockerfile
@@ -8,7 +8,8 @@ LABEL maintainer="haobibo@gmail.com"
 
 COPY work /opt/utils/
 
-RUN source /opt/utils/script-utils.sh && install_apt /opt/utils/install_list_openresty.apt \
+RUN set -ex \
+ && source /opt/utils/script-utils.sh && install_apt /opt/utils/install_list_openresty.apt \
  && source /opt/utils/script-setup.sh           && setup_lua_base && setup_lua_rocks \
  && source /opt/utils/script-setup-openresty.sh && setup_openresty \
  && source /opt/utils/script-setup-acme.sh      && setup_acme \