Merge branch 'main' into pytorch-image

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,6 +1,5 @@
 name: Bug report
 description: Create a report to help us improve
-title: "[BUG] - <title>"
 labels: ["type:Bug"]
 
 body:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,6 +1,5 @@
 name: Feature request
 description: Suggest a new feature for this project
-title: "[ENH] - <title>"
 labels: ["type:Enhancement"]
 
 body:

.github/workflows/contributed-recipes.yml

@@ -26,7 +26,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Calculate recipes matrix 🛠
         id: set-matrix
@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build recipe 🛠
         # We're pulling here to avoid accidentally using image which migt be present on aarch64 self-hosted runner

.github/workflows/docker-build-test-upload.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create dev environment 📦
         uses: ./.github/actions/create-dev-env
         with:

.github/workflows/docker-merge-tags.yml

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create dev environment 📦
         uses: ./.github/actions/create-dev-env
         with:
@@ -50,7 +50,7 @@ jobs:
 
       - name: Login to Docker Hub 🔐
         if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-tag-push.yml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create dev environment 📦
         uses: ./.github/actions/create-dev-env
         with:
@@ -39,7 +39,7 @@ jobs:
 
       - name: Login to Docker Hub 🔐
         if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-wiki-update.yml

@@ -11,7 +11,7 @@ jobs:
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Create dev environment 📦
         uses: ./.github/actions/create-dev-env
         with:
@@ -29,7 +29,7 @@ jobs:
         shell: bash
 
       - name: Checkout Wiki Repo 📃
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}.wiki
           path: wiki/

.github/workflows/hub-overview.yml

@@ -11,16 +11,17 @@ on:
       - ".github/workflows/hub-overview.yml"
 
       - "images/*/README.md"
+  workflow_dispatch:
 
 jobs:
-  update-dockerhub-overview:
-    name: Update Docker Hub overviews
+  update-overview:
     runs-on: ubuntu-latest
+    name: update-overview (${{matrix.image}})
     if: github.repository == 'jupyter/docker-stacks'
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Push README to Docker Hub 🐳
         uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1

.github/workflows/pre-commit.yml

@@ -12,12 +12,11 @@ permissions:
 
 jobs:
   run-hooks:
-    name: Run pre-commit hooks
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set Up Python 🐍
         uses: actions/setup-python@v4

.github/workflows/sphinx.yml

@@ -24,15 +24,14 @@ on:
 
 jobs:
   build-docs:
-    name: Build Sphinx Documentation
     permissions:
       contents: write
     if: github.repository == 'jupyter/docker-stacks' || github.event_name != 'schedule'
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout Repo ⚡️
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.pre-commit-config.yaml

@@ -14,7 +14,7 @@
 repos:
   # Autoupdate: Python code
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.10.1
+    rev: v3.13.0
     hooks:
       - id: pyupgrade
         args: [--py39-plus]
@@ -28,7 +28,7 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
       - id: black
         args: [--target-version=py39]
@@ -52,7 +52,7 @@ repos:
 
   # Autoformat: YAML, JSON, Markdown, etc.
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.2
+    rev: v3.0.3
     hooks:
       - id: prettier
 
@@ -113,7 +113,7 @@ repos:
 
   # Lint: Markdown
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.35.0
+    rev: v0.37.0
     hooks:
       - id: markdownlint
         args: ["--fix"]

README.md

@@ -26,11 +26,11 @@ The [User Guide on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/
 
 ### Example 1
 
-This command pulls the `jupyter/scipy-notebook` image tagged `2023-08-19` from Docker Hub if it is not already present on the local host.
+This command pulls the `jupyter/scipy-notebook` image tagged `2023-09-25` from Docker Hub if it is not already present on the local host.
 It then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the container's internal port `8888` to port `10000` of the host machine:
 
 ```bash
-docker run -p 10000:8888 jupyter/scipy-notebook:2023-08-19
+docker run -p 10000:8888 jupyter/scipy-notebook:2023-09-25
 ```
 
 You can modify the port on which the container's port is exposed by [changing the value of the `-p` option](https://docs.docker.com/engine/reference/run/#expose-incoming-ports) to `-p 8888:8888`.
@@ -45,11 +45,11 @@ The container remains intact for restart after the Server exits.
 
 ### Example 2
 
-This command pulls the `jupyter/datascience-notebook` image tagged `2023-08-19` from Docker Hub if it is not already present on the local host.
+This command pulls the `jupyter/datascience-notebook` image tagged `2023-09-25` from Docker Hub if it is not already present on the local host.
 It then starts an _ephemeral_ container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 10000.
 
 ```bash
-docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work jupyter/datascience-notebook:2023-08-19
+docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work jupyter/datascience-notebook:2023-09-25
 ```
 
 The use of the `-v` flag in the command mounts the current working directory on the host (`${PWD}` in the example command) as `/home/jovyan/work` in the container.
@@ -119,7 +119,7 @@ more information is available in the [documentation](https://jupyter-docker-stac
 - We publish containers for both `x86_64` and `aarch64` platforms
 - Single-platform images have either `aarch64-` or `x86_64-` tag prefixes, for example, `jupyter/base-notebook:aarch64-python-3.10.5`
 - Starting from `2022-09-21`, we create multi-platform images (except `tensorflow-notebook`)
-- Starting from `2023-08-19`, we create multi-platform `tensorflow-notebook` image as well
+- Starting from `2023-09-25`, we create multi-platform `tensorflow-notebook` image as well
 
 ## Using old images
 

binder/Dockerfile

@@ -3,7 +3,7 @@
 
 # https://hub.docker.com/r/jupyter/base-notebook/tags
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/base-notebook:2023-08-19
+ARG BASE_CONTAINER=$OWNER/base-notebook:2023-09-25
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
@@ -12,6 +12,6 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-ENV TAG="2023-08-19"
+ENV TAG="2023-09-25"
 
 COPY --chown=${NB_UID}:${NB_GID} binder/README.ipynb "${HOME}"/README.ipynb

docs/using/running.md

@@ -15,12 +15,12 @@ The following are some common patterns.
 
 ### Example 1
 
-This command pulls the `jupyter/scipy-notebook` image tagged `2023-08-19` from Docker Hub if it is not already present on the local host.
+This command pulls the `jupyter/scipy-notebook` image tagged `2023-09-25` from Docker Hub if it is not already present on the local host.
 It then starts a container running Jupyter Server with the JupyterLab frontend and exposes the server on host port 8888.
 The server logs appear in the terminal and include a URL to the server.
 
 ```bash
-docker run -it -p 8888:8888 jupyter/scipy-notebook:2023-08-19
+docker run -it -p 8888:8888 jupyter/scipy-notebook:2023-09-25
 
 # Entered start.sh with args: jupyter lab
 
@@ -39,7 +39,7 @@ Pressing `Ctrl-C` twice shuts down the Server but leaves the container intact on
 # list containers
 docker ps --all
 # CONTAINER ID   IMAGE                                                 COMMAND                  CREATED          STATUS                     PORTS     NAMES
-# 221331c047c4   jupyter/scipy-notebook:2023-08-19                   "tini -g -- start-no…"   11 seconds ago   Exited (0) 8 seconds ago             cranky_benz
+# 221331c047c4   jupyter/scipy-notebook:2023-09-25                   "tini -g -- start-no…"   11 seconds ago   Exited (0) 8 seconds ago             cranky_benz
 
 # start the stopped container
 docker start --attach 221331c047c4
@@ -53,12 +53,12 @@ docker rm 221331c047c4
 
 ### Example 2
 
-This command pulls the `jupyter/r-notebook` image tagged `2023-08-19` from Docker Hub if it is not already present on the local host.
+This command pulls the `jupyter/r-notebook` image tagged `2023-09-25` from Docker Hub if it is not already present on the local host.
 It then starts a container running Server and exposes the server on host port 10000.
 The server logs appear in the terminal and include a URL to the Server, but with the internal container port (8888) instead of the correct host port (10000).
 
 ```bash
-docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work jupyter/r-notebook:2023-08-19
+docker run -it --rm -p 10000:8888 -v "${PWD}":/home/jovyan/work jupyter/r-notebook:2023-09-25
 ```
 
 Pressing `Ctrl-C` twice shuts down the Server and immediately destroys the Docker container.
@@ -137,7 +137,7 @@ subuidSize=$(( $(podman info --format "{{ range .Host.IDMappings.UIDMap }}+{{.Si
 subgidSize=$(( $(podman info --format "{{ range .Host.IDMappings.GIDMap }}+{{.Size }}{{end }}" ) - 1 ))
 ```
 
-This command pulls the `docker.io/jupyter/r-notebook` image tagged `2023-08-19` from Docker Hub if it is not already present on the local host.
+This command pulls the `docker.io/jupyter/r-notebook` image tagged `2023-09-25` from Docker Hub if it is not already present on the local host.
 It then starts a container running a Jupyter Server with the JupyterLab frontend and exposes the server on host port 10000.
 The server logs appear in the terminal and include a URL to the server, but with the internal container port (8888) instead of the correct host port (10000).
 
@@ -146,7 +146,7 @@ podman run -it --rm -p 10000:8888 \
     -v "${PWD}":/home/jovyan/work --user $uid:$gid \
     --uidmap $uid:0:1 --uidmap 0:1:$uid --uidmap $(($uid+1)):$(($uid+1)):$(($subuidSize-$uid)) \
     --gidmap $gid:0:1 --gidmap 0:1:$gid --gidmap $(($gid+1)):$(($gid+1)):$(($subgidSize-$gid)) \
-    docker.io/jupyter/r-notebook:2023-08-19
+    docker.io/jupyter/r-notebook:2023-09-25
 ```
 
 ```{warning}

docs/using/selecting.md

@@ -120,7 +120,7 @@ It contains:
 `jupyter/julia-notebook` includes popular packages from the Julia ecosystem listed below:
 
 - Everything in `jupyter/minimal-notebook` and its ancestor images
-- The [Julia Programming Language](https://julialang.org/)
+- The [Julia](https://julialang.org/) compiler and base environment
 - [IJulia](https://github.com/JuliaLang/IJulia.jl) to support Julia code in Jupyter notebook
 - [Pluto.jl](https://plutojl.org/) reactive Julia notebook interface, made accessible with [jupyter-pluto-proxy](https://github.com/yuvipanda/jupyter-pluto-proxy)
 - [HDF5](https://github.com/JuliaIO/HDF5.jl) package
@@ -197,16 +197,11 @@ It contains:
 [Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/images/datascience-notebook/Dockerfile) |
 [Docker Hub image tags](https://hub.docker.com/r/jupyter/datascience-notebook/tags/)
 
-`jupyter/datascience-notebook` includes libraries for data analysis from the Julia, Python, and R
-communities.
+`jupyter/datascience-notebook` includes libraries for data analysis from the Python, and R, and Julia communities.
 
-- Everything in the `jupyter/scipy-notebook` and `jupyter/r-notebook` images and their ancestor
+- Everything in the `jupyter/scipy-notebook`, `jupyter/r-notebook`, and `jupyter/julia-notebook` images and their ancestor
   images
 - [rpy2](https://rpy2.github.io/doc/latest/html/index.html) package
-- The [Julia](https://julialang.org/) compiler and base environment
-- [IJulia](https://github.com/JuliaLang/IJulia.jl) to support Julia code in Jupyter notebooks
-- [Pluto.jl](https://plutojl.org/) reactive Julia notebook interface, made accessible with [jupyter-pluto-proxy](https://github.com/yuvipanda/jupyter-pluto-proxy)
-- [HDF5](https://github.com/JuliaIO/HDF5.jl) package
 
 ### jupyter/pyspark-notebook
 

images/docker-stacks-foundation/run-hooks.sh

@@ -24,11 +24,19 @@ for f in "${1}/"*; do
             echo "Sourcing shell script: ${f}"
             # shellcheck disable=SC1090
             source "${f}"
+            # shellcheck disable=SC2181
+            if [ $? -ne 0 ] ; then
+                echo "${f} has failed, continuing execution"
+            fi
             ;;
         *)
             if [ -x "${f}" ] ; then
                 echo "Running executable: ${f}"
                 "${f}"
+                # shellcheck disable=SC2181
+                if [ $? -ne 0 ] ; then
+                    echo "${f} has failed, continuing execution"
+                fi
             else
                 echo "Ignoring non-executable: ${f}"
             fi

images/minimal-notebook/setup-scripts/setup-julia.bash

@@ -6,7 +6,7 @@ set -exuo pipefail
 
 # Default julia version to install if env var is not set
 # Check https://julialang.org/downloads/
-JULIA_VERSION="${JULIA_VERSION:-1.9.2}"
+JULIA_VERSION="${JULIA_VERSION:-1.9.3}"
 
 # Figure out what architecture we are installing in
 JULIA_ARCH=$(uname -m)

images/pyspark-notebook/Dockerfile

@@ -15,10 +15,10 @@ USER root
 # Spark dependencies
 # Default values can be overridden at build time
 # (ARGS are in lower case to distinguish them from ENV)
-ARG spark_version="3.4.1"
+ARG spark_version="3.5.0"
 ARG hadoop_version="3"
 ARG scala_version
-ARG spark_checksum="5a21295b4c3d1d3f8fc85375c711c7c23e3eeb3ec9ea91778f149d8d321e3905e2f44cf19c69a28df693cffd536f7316706c78932e7e148d224424150f18b2c5"
+ARG spark_checksum="8883c67e0a138069e597f3e7d4edbbd5c3a565d50b28644aad02856a1ec1da7cb92b8f80454ca427118f69459ea326eaa073cf7b1a860c3b796f4b07c2101319"
 ARG openjdk_version="17"
 
 ENV APACHE_SPARK_VERSION="${spark_version}" \
@@ -66,9 +66,14 @@ RUN fix-permissions "/etc/ipython/"
 USER ${NB_UID}
 
 # Install pyarrow
-# Temporarily pin pandas to version 1.5.3, see: https://github.com/jupyter/docker-stacks/issues/1924
+# NOTE: It's important to ensure compatibility between Pandas versions.
+# The pandas version in this Dockerfile should match the version
+# on which the Pandas API for Spark is built.
+# To find the right version:
+# 1. Check out the Spark branch you are on.
+# 2. Find the pandas version in the file spark/dev/infra/Dockerfile.
 RUN mamba install --yes \
-    'pandas>=1.5.3,<2.0.0' \
+    'pandas=2.0.3' \
     'pyarrow' && \
     mamba clean --all -f -y && \
     fix-permissions "${CONDA_DIR}" && \

tests/docker-stacks-foundation/run-hooks-failures/a.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+echo "Started: a.sh"
+
+export OTHER_VAR=456
+
+run-unknown-command
+
+echo "Finished: a.sh"