From 62fefa12ec2a86bace4a13ffbf8e120e8f26dea3 Mon Sep 17 00:00:00 2001 From: Susanne Reinsbach <61008432+susrei@users.noreply.github.com> Date: Sat, 23 Dec 2023 13:45:47 +0100 Subject: [PATCH] feat: add containers (#26) * add dotfiles to gitignore * add build-and-push gh action per toolkit * add dockerfiles and install scripts * add launch scripts and def files for uppmax * add qmd docs for workshop * add readme for containers folder * rename containers docs qmd file * udpate readme with new image names --- .../docker-publish-bioconductor-spatial.yaml | 67 +++++++ .../docker-publish-bioconductor.yaml | 67 +++++++ .../docker-publish-scanpy-spatial.yaml | 67 +++++++ .github/workflows/docker-publish-scanpy.yaml | 67 +++++++ .../docker-publish-seurat-spatial.yaml | 67 +++++++ .github/workflows/docker-publish-seurat.yaml | 67 +++++++ .gitignore | 6 + README.md | 24 +-- containers/README.md | 130 +++++++++++++ .../dockerfiles/bioconductor.Dockerfile | 17 ++ .../bioconductor_spatial.Dockerfile | 15 ++ containers/dockerfiles/scanpy.Dockerfile | 32 +++ .../dockerfiles/scanpy_spatial.Dockerfile | 28 +++ containers/dockerfiles/seurat.Dockerfile | 17 ++ .../dockerfiles/seurat_spatial.Dockerfile | 15 ++ containers/docs/precourse-instructions.qmd | 60 ++++++ containers/docs/run-containers.qmd | 183 ++++++++++++++++++ containers/scripts/install_bioconductor.sh | 80 ++++++++ .../scripts/install_bioconductor_spatial.sh | 56 ++++++ containers/scripts/install_scanpy.sh | 37 ++++ containers/scripts/install_scanpy_spatial.sh | 37 ++++ containers/scripts/install_seurat.sh | 75 +++++++ containers/scripts/install_seurat_spatial.sh | 60 ++++++ .../scripts/reticulate_install_bioconductor.R | 12 ++ .../reticulate_install_bioconductor_spatial.R | 8 + .../scripts/reticulate_install_seurat.R | 13 ++ .../reticulate_install_seurat_spatial.R | 8 + containers/uppmax/launch_jupyter.sh | 48 +++++ containers/uppmax/launch_rstudio.sh | 84 ++++++++ .../uppmax/singularity_bioconductor.def | 7 + containers/uppmax/singularity_scanpy.def | 5 + containers/uppmax/singularity_seurat.def | 7 + 32 files changed, 1454 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/docker-publish-bioconductor-spatial.yaml create mode 100644 .github/workflows/docker-publish-bioconductor.yaml create mode 100644 .github/workflows/docker-publish-scanpy-spatial.yaml create mode 100644 .github/workflows/docker-publish-scanpy.yaml create mode 100644 .github/workflows/docker-publish-seurat-spatial.yaml create mode 100644 .github/workflows/docker-publish-seurat.yaml create mode 100644 containers/README.md create mode 100644 containers/dockerfiles/bioconductor.Dockerfile create mode 100644 containers/dockerfiles/bioconductor_spatial.Dockerfile create mode 100644 containers/dockerfiles/scanpy.Dockerfile create mode 100644 containers/dockerfiles/scanpy_spatial.Dockerfile create mode 100644 containers/dockerfiles/seurat.Dockerfile create mode 100644 containers/dockerfiles/seurat_spatial.Dockerfile create mode 100644 containers/docs/precourse-instructions.qmd create mode 100644 containers/docs/run-containers.qmd create mode 100755 containers/scripts/install_bioconductor.sh create mode 100755 containers/scripts/install_bioconductor_spatial.sh create mode 100755 containers/scripts/install_scanpy.sh create mode 100755 containers/scripts/install_scanpy_spatial.sh create mode 100755 containers/scripts/install_seurat.sh create mode 100755 containers/scripts/install_seurat_spatial.sh create mode 100755 containers/scripts/reticulate_install_bioconductor.R create mode 100755 containers/scripts/reticulate_install_bioconductor_spatial.R create mode 100755 containers/scripts/reticulate_install_seurat.R create mode 100755 containers/scripts/reticulate_install_seurat_spatial.R create mode 100755 containers/uppmax/launch_jupyter.sh create mode 100755 containers/uppmax/launch_rstudio.sh create mode 100644 containers/uppmax/singularity_bioconductor.def create mode 100644 containers/uppmax/singularity_scanpy.def create mode 100644 containers/uppmax/singularity_seurat.def diff --git a/.github/workflows/docker-publish-bioconductor-spatial.yaml b/.github/workflows/docker-publish-bioconductor-spatial.yaml new file mode 100644 index 00000000..6ea68792 --- /dev/null +++ b/.github/workflows/docker-publish-bioconductor-spatial.yaml @@ -0,0 +1,67 @@ +name: Build and Push Bioconductor-Spatial + +on: + push: + branches: + - "master" + paths: + - "containers/dockerfiles/bioconductor_spatial.Dockerfile" + - "containers/scripts/install_bioconductor_spatial.sh" + - "containers/scripts/reticulate_install_bioconductor_spatial.R" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: 2023-bioconductor_spatial-r4.3.0 + + +jobs: + bioconductor_spatial: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./containers + file: containers/dockerfiles/bioconductor_spatial.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/docker-publish-bioconductor.yaml b/.github/workflows/docker-publish-bioconductor.yaml new file mode 100644 index 00000000..5746ea79 --- /dev/null +++ b/.github/workflows/docker-publish-bioconductor.yaml @@ -0,0 +1,67 @@ +name: Build and Push Bioconductor + +on: + push: + branches: + - "master" + paths: + - "containers/dockerfiles/bioconductor.Dockerfile" + - "containers/scripts/install_bioconductor.sh" + - "containers/scripts/reticulate_install_bioconductor.R" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: 2023-bioconductor-r4.3.0 + + +jobs: + bioconductor: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./containers + file: containers/dockerfiles/bioconductor.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/docker-publish-scanpy-spatial.yaml b/.github/workflows/docker-publish-scanpy-spatial.yaml new file mode 100644 index 00000000..d117ec5c --- /dev/null +++ b/.github/workflows/docker-publish-scanpy-spatial.yaml @@ -0,0 +1,67 @@ +name: Build and Push Scanpy-Spatial + +on: + push: + branches: + - "master" + paths: + - "containers/dockerfiles/scanpy_spatial.Dockerfile" + - "containers/scripts/install_scanpy_spatial.sh" + - "containers/scripts/reticulate_install_scanpy_spatial.R" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: 2023-scanpy_spatial-py3.10 + + +jobs: + scanpy_spatial: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./containers + file: containers/dockerfiles/scanpy_spatial.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/docker-publish-scanpy.yaml b/.github/workflows/docker-publish-scanpy.yaml new file mode 100644 index 00000000..3a62e849 --- /dev/null +++ b/.github/workflows/docker-publish-scanpy.yaml @@ -0,0 +1,67 @@ +name: Build and Push Scanpy + +on: + push: + branches: + - "master" + paths: + - "containers/dockerfiles/scanpy.Dockerfile" + - "containers/scripts/install_scanpy.sh" + - "containers/scripts/reticulate_install_scanpy.R" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: 2023-scanpy-py3.10 + + +jobs: + scanpy: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./containers + file: containers/dockerfiles/scanpy.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/docker-publish-seurat-spatial.yaml b/.github/workflows/docker-publish-seurat-spatial.yaml new file mode 100644 index 00000000..439474eb --- /dev/null +++ b/.github/workflows/docker-publish-seurat-spatial.yaml @@ -0,0 +1,67 @@ +name: Build and Push Seurat-Spatial + +on: + push: + branches: + - "master" + paths: + - "containers/dockerfiles/seurat_spatial.Dockerfile" + - "containers/scripts/install_seurat_spatial.sh" + - "containers/scripts/reticulate_install_seurat_spatial.R" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: 2023-seurat_spatial-r4.3.0 + + +jobs: + seurat_spatial: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./containers + file: containers/dockerfiles/seurat_spatial.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.github/workflows/docker-publish-seurat.yaml b/.github/workflows/docker-publish-seurat.yaml new file mode 100644 index 00000000..a218e7bf --- /dev/null +++ b/.github/workflows/docker-publish-seurat.yaml @@ -0,0 +1,67 @@ +name: Build and Push Seurat + +on: + push: + branches: + - "master" + paths: + - "containers/dockerfiles/seurat.Dockerfile" + - "containers/scripts/install_seurat.sh" + - "containers/scripts/reticulate_install_seurat.R" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: 2023-seurat-r4.3.0 + + +jobs: + seurat: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install cosign + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: ./containers + file: containers/dockerfiles/seurat.Dockerfile + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} + labels: ${{ steps.meta.outputs.labels }} + push: true + + - name: Sign the published Docker image + env: + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} diff --git a/.gitignore b/.gitignore index 711411a8..eeaab816 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,9 @@ **/sinfo-requirements.txt **/.vdoc.py **/.pybiomart.sqlite +.bash_history +.cache/ +.ipython/ +.jupyter/ +.local/ +.npm/ \ No newline at end of file diff --git a/README.md b/README.md index ba88ca78..ae8a3dd4 100644 --- a/README.md +++ b/README.md @@ -22,33 +22,33 @@ git push origin ``` # for seurat labs -docker pull susrei/workshop-scrnaseq:2023-seurat-r4.3.0-conda-231204 +docker pull ghcr.io/NBISweden/workshop-scRNAseq:2023-seurat-r4.3.0 # for bioconductor labs -docker pull susrei/workshop-scrnaseq:2023-bioconductor-r4.3.0-conda-231204 +docker pull ghcr.io/NBISweden/workshop-scRNAseq:2023-bioconductor-r4.3.0 # for python labs -docker pull susrei/workshop-scrnaseq:2023-scanpy-py3.10 +docker pull ghcr.io/NBISweden/workshop-scRNAseq:2023-scanpy-py3.10 ``` -## Run labs +## Run labs (locally) -- Launch docker container in root of the repo +- Launch docker container in the project's root folder - To run Seurat or Bioconductor labs in RStudio ``` -docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir susrei/workshop-scrnaseq:2023-seurat-r4.3.0-conda-231204 -docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir susrei/workshop-scrnaseq:2023-bioconductor-r4.3.0-conda-231204 +docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-seurat-r4.3.0 +docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-bioconductor-r4.3.0 ``` - Open in browser: `http://localhost:8787/`, login: rstudio, pass: scrnaseq - Navigate to `/home/rstudio/workdir/labs` and open qmd files -- To run Python labs in Jupyter notebook +- To run Python labs in JupyterLab ``` -docker run --rm -ti --platform=linux/amd64 -p 8888:8888 -v $PWD:/home/jovyan/workdir susrei/workshop-scrnaseq:2023-scanpy-py3.10 +docker run --rm -ti --platform=linux/amd64 -p 8888:8888 -v $PWD:/home/jovyan/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-scanpy-py3.10 ``` - Open in browser: `http://127.0.0.1:8888/lab?token=xxxx` (Use exact token from terminal on launch) @@ -61,19 +61,19 @@ Instructions to render the `.qmd` files to `.html`. - For Seurat labs ``` -docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir susrei/workshop-scrnaseq:2023-seurat-r4.3.0-conda-231204 quarto render /home/rstudio/workdir/labs/seurat/seurat_01_qc.qmd +docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-seurat-r4.3.0 quarto render /home/rstudio/workdir/labs/seurat/seurat_01_qc.qmd ``` - For Bioconductor labs ``` -docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir susrei/workshop-scrnaseq:2023-bioconductor-r4.3.0-conda-231204 quarto render /home/rstudio/workdir/labs/bioc/bioc_01_qc.qmd +docker run --rm -ti --platform=linux/amd64 -p 8787:8787 -e PASSWORD=scrnaseq -v $PWD:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-bioconductor-r4.3.0 quarto render /home/rstudio/workdir/labs/bioc/bioc_01_qc.qmd ``` - For Python labs ``` -docker run --rm -ti --platform=linux/amd64 -p 8888:8888 -v $PWD:/home/jovyan/workdir susrei/workshop-scrnaseq:2023-scanpy-py3.10 quarto render /home/jovyan/workdir/labs/scanpy/scanpy_01_qc.qmd +docker run --rm -ti --platform=linux/amd64 -p 8888:8888 -v $PWD:/home/jovyan/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-scanpy-py3.10 quarto render /home/jovyan/workdir/labs/scanpy/scanpy_01_qc.qmd ``` - Successfully rendered outputs are moved to `docs` folder and chunks are cached under `_freeze`. diff --git a/containers/README.md b/containers/README.md new file mode 100644 index 00000000..10ac367a --- /dev/null +++ b/containers/README.md @@ -0,0 +1,130 @@ +# Docker Images + +The Docker images are based on [rocker-org/rocker-versioned2](https://github.com/rocker-org/rocker-versioned2) images, and follow the same folder structure. + +The R images are based on the `rocker/tidyverse` with a Dockerfile per toolkit and the corresponding install scripts in the `scripts/` directory. The Python image is based on the `jupyter/minimal-notebook`. + +> **NOTE:** All images contain `quarto` command line utility. + +## Description + +Here we use three different toolkits, namely `Seurat` (R/RStudio), `Bioconductor` (R/RStudio) and `Scanpy` (Python/Jupyter) to perform scRNAseq analysis provided in the NBIS scRNAseq workshop. + +The different images are differentiated by the following `registry/username/image:tag` convention: + +``` +ghcr.io/NBISweden/workshop-scRNAseq:-- +``` + +Each image contains the required packages so that, for each toolkit, the following analysis steps can be performed: +* QC +* Dimensionality reduction +* Data integration +* Clustering +* Differential expression +* Celltype prediction +* Trajectory analysis + +## Working with Python from R + +For the R based images, a Python environment is created using the `reticulate` R package. + +> **NOTE:** It is important to know that depending on the package version and it's dependencies, some packages are installed in a `conda` environment, while others are installed in a `venv` environment. + +For `Bioconductor`, the Python environment includes `scanorama`. To use this environment, add the following lines in your notebook: + +```R +library(reticulate) +use_virtualenv("/opt/venv/scanorama") +py_discover_config() +``` + +Additionally, a `conda` environment is available and can be activated by adding the following lines to your notebook: + +```R +library(reticulate) +use_condaenv("bioc", conda = "/opt/conda/bin/conda") +py_discover_config() +``` + +For `Seurat`, the `conda` environment includes `umap-learn`. To use this environment, add the following lines in your notebook: + +```R +library(reticulate) +use_condaenv("seurat", conda = "/opt/conda/bin/conda") +py_discover_config() +``` + +The `scanorama` package is available in the Python environment and can be activated by adding the following lines: + +```R +library(reticulate) +use_virtualenv("/opt/venv/scanorama") +py_discover_config() +``` + +## Installing additional R packages + +> **NOTE:** These packages will be installed in the *running container*, and are not persisted when the container is terminated. + +If you need to install additional R packages from `Cran`, `Bioconductor` or `GitHub`, the following scripts are available for convenience. From the _Terminal_, run the corresponding command below: + +* CRAN + ```bash + install2.r --error --skipinstalled -n "$NCPUS" + ``` + +* Bioconductor + ```bash + installBioc.r --error --skipinstalled -n "$NCPUS" + ``` + +* GitHub + ```bash + installGithub.r / + ``` + +## Installing additional Python packages + +> **NOTE:** These packages will be installed in the *running container*, and are not persisted when the container is terminated. + +For the Python based image, you can install packages using `mamba`, `pip`, or `conda` (`mamba` is recommended). + +## How To Run + +### R Based Images + +```bash +docker pull ghcr.io/NBISweden/workshop-scRNAseq: +docker run --rm -ti -p 8787:8787 -e PASSWORD=scrnaseq -v /path/to/workdir:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq: +``` + +In the browser, go to [localhost:8787](localhost:8787). +Use the following credentials to log in to the RStudio Server: +> user: rstudio +> password: scrnaseq + +### Python Based Image + +```bash +docker pull ghcr.io/NBISweden/workshop-scRNAseq: +docker run --rm -ti -p 8888:8888 -v /path/to/workdir:/home/jovyan/workshop-scRNAseq ghcr.io/NBISweden/workshop-scRNAseq: +``` + +In the browser, go to [localhost:8888](localhost:8888). + +## How To Build + +From the project root directory, run the following command: + +```bash +docker build -t scrnaseq: --file containers/dockerfiles/.Dockerfile ./containers +``` + +## Push to GitHub Container Registry + +Each toolkit image comes with an associated GitHub Action that builds and pushes the image to GitHub Container Registry (`ghcr.io`). Each workflow is triggered only on changes to files related to corresponding toolkit image. + +## Build Singularity Images (Uppmax) + +To use these images on Uppmax, you first need to build them as Singularity images. A set of minimal Singularity definition files (`.def`) are provided in the `uppmax/` folder. A set of launch scripts, one for JupyterLab based images and one for RStudio based images are also provided in the same folder. diff --git a/containers/dockerfiles/bioconductor.Dockerfile b/containers/dockerfiles/bioconductor.Dockerfile new file mode 100644 index 00000000..718acc9d --- /dev/null +++ b/containers/dockerfiles/bioconductor.Dockerfile @@ -0,0 +1,17 @@ +FROM rocker/tidyverse:4.3.0 + +ARG NCPUS=${NCPUS:--1} + +COPY scripts/install_bioconductor.sh \ + scripts/reticulate_install_bioconductor.R \ + /rocker_scripts/ + +RUN /rocker_scripts/install_bioconductor.sh + +RUN Rscript /rocker_scripts/reticulate_install_bioconductor.R + +## Make user rstudio owner of the Python environment dirs +RUN chgrp -R rstudio /opt/conda && \ + sudo chmod 770 -R /opt/conda +RUN chgrp -R rstudio /opt/venv && \ + sudo chmod 770 -R /opt/venv diff --git a/containers/dockerfiles/bioconductor_spatial.Dockerfile b/containers/dockerfiles/bioconductor_spatial.Dockerfile new file mode 100644 index 00000000..019c5e01 --- /dev/null +++ b/containers/dockerfiles/bioconductor_spatial.Dockerfile @@ -0,0 +1,15 @@ +FROM rocker/tidyverse:4.3.2 + +ARG NCPUS=${NCPUS:--1} + +COPY scripts/install_bioconductor_spatial.sh \ + scripts/reticulate_install_bioconductor_spatial.R \ + /rocker_scripts/ + +RUN /rocker_scripts/install_bioconductor_spatial.sh + +RUN Rscript /rocker_scripts/reticulate_install_bioconductor_spatial.R + +## Make user rstudio owner of the Python environment dirs +RUN chgrp -R rstudio /opt/conda && \ + sudo chmod 770 -R /opt/conda diff --git a/containers/dockerfiles/scanpy.Dockerfile b/containers/dockerfiles/scanpy.Dockerfile new file mode 100644 index 00000000..fdf7b7ed --- /dev/null +++ b/containers/dockerfiles/scanpy.Dockerfile @@ -0,0 +1,32 @@ +FROM jupyter/minimal-notebook:python-3.10 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG TARGETARCH + +USER root + +COPY scripts/install_scanpy.sh /scripts/ + +RUN /scripts/install_scanpy.sh + +RUN mamba install --yes --channel conda-forge --channel bioconda \ + bbknn=1.6.0 \ + gseapy=1.0.6 \ + matplotlib-venn=0.11.9 \ + openpyxl=3.1.2 \ + pybiomart=0.2.0 \ + scanorama=1.7.4 \ + scanpy=1.9.6 \ + scrublet=0.2.3 \ + scvi-tools=1.0.4 && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir \ + leidenalg==0.10.1 \ + louvain==0.8.1 + +USER ${NB_UID} diff --git a/containers/dockerfiles/scanpy_spatial.Dockerfile b/containers/dockerfiles/scanpy_spatial.Dockerfile new file mode 100644 index 00000000..3649a95d --- /dev/null +++ b/containers/dockerfiles/scanpy_spatial.Dockerfile @@ -0,0 +1,28 @@ +FROM jupyter/minimal-notebook:python-3.10 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG TARGETARCH + +USER root + +COPY scripts/install_scanpy.sh /scripts/ + +RUN /scripts/install_scanpy.sh + +RUN mamba install --yes --channel conda-forge --channel bioconda --channel anaconda \ + scanorama=1.7.4 \ + scanpy=1.9.6 \ + scvi-tools=1.0.4 \ + seaborn=0.12.2 && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir \ + leidenalg==0.10.1 \ + louvain==0.8.1 \ + spatialde==1.1.3 + +USER ${NB_UID} diff --git a/containers/dockerfiles/seurat.Dockerfile b/containers/dockerfiles/seurat.Dockerfile new file mode 100644 index 00000000..c89c4dc9 --- /dev/null +++ b/containers/dockerfiles/seurat.Dockerfile @@ -0,0 +1,17 @@ +FROM rocker/tidyverse:4.3.0 + +ARG NCPUS=${NCPUS:--1} + +COPY scripts/install_seurat.sh \ + scripts/reticulate_install_seurat.R \ + /rocker_scripts/ + +RUN /rocker_scripts/install_seurat.sh + +RUN Rscript /rocker_scripts/reticulate_install_seurat.R + +## Make user rstudio owner of the Python environment dirs +RUN chgrp -R rstudio /opt/conda && \ + sudo chmod 770 -R /opt/conda +RUN chgrp -R rstudio /opt/venv && \ + sudo chmod 770 -R /opt/venv diff --git a/containers/dockerfiles/seurat_spatial.Dockerfile b/containers/dockerfiles/seurat_spatial.Dockerfile new file mode 100644 index 00000000..88816a4d --- /dev/null +++ b/containers/dockerfiles/seurat_spatial.Dockerfile @@ -0,0 +1,15 @@ +FROM rocker/tidyverse:4.3.2 + +ARG NCPUS=${NCPUS:--1} + +COPY scripts/install_seurat_spatial.sh \ + scripts/reticulate_install_seurat_spatial.R \ + /rocker_scripts/ + +RUN /rocker_scripts/install_seurat_spatial.sh + +RUN Rscript /rocker_scripts/reticulate_install_seurat_spatial.R + +## Make user rstudio owner of the Python environment dirs +RUN chgrp -R rstudio /opt/conda && \ + sudo chmod 770 -R /opt/conda diff --git a/containers/docs/precourse-instructions.qmd b/containers/docs/precourse-instructions.qmd new file mode 100644 index 00000000..25e01d52 --- /dev/null +++ b/containers/docs/precourse-instructions.qmd @@ -0,0 +1,60 @@ +# Docker + +## Install Docker Desktop + +Ensure that you install Docker Desktop before the course starts. If you do not have admin rights to install software on your lapop, talk to your local IT for help. + + +### Linux + +1. Follow the installation instructions for your OS: + + - [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) + - [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) + - [Fedora](https://docs.docker.com/install/linux/docker-ce/fedora/) + +2. After Docker Desktop starts, open the Dashboard and go to _Settings (⚙️ in the top-right) > General_. Follow the [instructions](https://docs.docker.com/desktop/settings/linux/) in the _General_ section. + + +### Mac + +1. Visit the [this page](https://docs.docker.com/docker-for-mac/install/#what-to-know-before-you-install) to ensure you have the requirements necessary. + +2. Download [Docker Desktopfor Mac](https://hub.docker.com/editions/community/docker-ce-desktop-mac) with _Apple Silicon_ or _Intel Chip_ and follow the installation instructions. + +3. After Docker Desktop starts, open the Dashboard and go to _Settings (⚙️ in the top-right) > General_. Follow the [instructions](https://docs.docker.com/desktop/settings/mac/) in the _General_ section. + + +### Windows + +1. Visit [this page](https://docs.docker.com/desktop/install/windows-install/) to ensure you have the requirements necessary. + +2. Download [Docker Desktop for Windows](https://hub.docker.com/editions/community/docker-ce-desktop-windows) and follow the installation instructions. + +3. After Docker Desktop starts, open the Dashboard and go to _Settings (⚙️ in the top-right) > General_. Follow the [instructions](https://docs.docker.com/desktop/settings/windows//) in the _General_ section. + + +## Test installation + +From the terminal, type: + +```bash +docker --version +``` + +and then run your first image by typing: + +```bash +docker run hello-world +``` + +If both work as expected, you successfully installed Docker Desktop! + + +## Allocate resources + +Open the Docker Dashboard when Docker Desktop starts and go to _Settings (⚙️ in the top-right) > Resources_ to allocate the following resources: + - *CPU limit*: `8` + - *Memory limit*: `12 GB` + - *Swap*: `2 GB` + diff --git a/containers/docs/run-containers.qmd b/containers/docs/run-containers.qmd new file mode 100644 index 00000000..a8eaf23e --- /dev/null +++ b/containers/docs/run-containers.qmd @@ -0,0 +1,183 @@ +# How To Run the Labs + +Three different toolkits, namely **`Seurat`** (R/RStudio), **`Bioconductor`** (R/RStudio) and **`Scanpy`** (Python/Jupyter) are available to perform the scRNAseq analysis. The labs can be run on Uppmax using Singularity (Apptainer) or on your local machine using Docker. Both options provide the necessary environment to run the analysis. + +## **Option A:** Run Singularity on Uppmax (Recommended) + +### Launch RStudio + +::: {.callout-tip} +To avoid running out of memory, restart R (_Session > Restart R_) after each lab. +::: + + +#### Seurat + +To launch RStudio Server and run the `Seurat` labs perform the following steps: + +```bash +cd /path/to/workdir # replace this with the full path to the workshop folder +/sw/courses/scrnaseq/singularity/launch_rstudio.sh /sw/courses/scrnaseq/singularity/2023-seurat-r4.3.0.sif +``` + +#### Bioconductor + +To launch RStudio Server and run the `Bioconductor` labs perform the following steps: + +```bash +cd /path/to/workdir # replace this with the full path to the workshop folder +/sw/courses/scrnaseq/singularity/launch_rstudio.sh /sw/courses/scrnaseq/singularity/2023-bioconductor-r4.3.0.sif +``` + +### Connect to RStudio + +After executing the `launch_rstudio.sh` script, a message with *your* login credentials will be printed to your screen, and it looks similar to the one below. + +::: {.callout-important} +Do not close this terminal! +::: + +``` + ************************************************* + * * + * IMPORTANT: Do not close or exit this shell! * + * * + ************************************************* + +1. SSH tunnel from your workstation using the following command: + + ssh -N -L 8787:rackham3.uppmax.uu.se:58359 susanner@rackham3.uppmax.uu.se + + and point your web browser to http://localhost:8787 + +2. log in to RStudio Server using the following credentials: + + user: susanner + password: scrnaseq + +When done using RStudio Server, terminate the job by: + +1. Exit the RStudio Session ("power" button in the top right corner of the RStudio window). +2. Issue the following command in both shells: + + CTRL-C +``` + +Follow the instructions printed on the screen to launch the RStudio Server. Open a shell *locally* and run *your* `ssh` command from step 1. Then open [localhost:8787](localhost:8787) in your web browser and log in to the RStudio Server using *your* `user` and `password` provided in step 2. + +### Launch JupyterLab + +#### Scanpy + +To launch JupyterLab and run the `Scanpy` labs using Jupyter notebooks perform the following steps: + +::: {.callout-tip} +To avoid running out of memory, restart the kernel (_Kernel > Restart Kernel_) after each lab. +::: + +```bash +cd /path/to/workdir # replace this with the full path to the workshop folder +/sw/courses/scrnaseq/singularity/launch_jupyter.sh /sw/courses/scrnaseq/singularity/2023-scanpy-py3.10.sif +``` + +### Connect to JupyterLab + +After executing the `launch_jupyter.sh` script, a message with *your* login credentials will be printed to your screen, and it looks similar to the one below. + +::: {.callout-important} +Do not close this terminal! +::: + +``` + ************************************************* + * * + * IMPORTANT: Do not close or exit this shell! * + * * + ************************************************* + +1. SSH tunnel from your workstation using the following command: + + ssh -N -L 8888:rackham1.uppmax.uu.se:34968 susanner@rackham1.uppmax.uu.se + + point your web browser to http://localhost:8888/lab + +2. Log in to JupyterLab using the password: + + scrnaseq + +When done using JupyterLab, terminate the job by: + +1. Shut down all kernels. +2. Issue the following command in both shells: + + CTRL-C +``` + +Follow the instructions to launch the JupyterLab. Open a shell *locally* and run *your* `ssh` command from step 1. Then open [localhost:8888/lab](localhost:8888/lab) in your web browser and log in to the JupyterLab using the password provided in step 2. + + +## **Option B:** Run Docker Locally + +### Seurat + +::: {.callout-tip} +To avoid running out of memory, restart R (_Session > Restart R_) after each lab. +::: + +```bash +cd /path/to/workshop-dir # replace this with the full path to the workshop folder +docker pull ghcr.io/NBISweden/workshop-scRNAseq:2023-seurat-r4.3.0 +docker run --rm -ti -p 8787:8787 -e PASSWORD=scrnaseq -v ${PWD}:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-seurat-r4.3.0 +``` + +In the browser, go to [localhost:8787](localhost:8787). +Use the following credentials to log in to the RStudio Server: + +> User: `rstudio` +> Password: `scrnaseq` + +### Bioconductor + +::: {.callout-tip} +To avoid running out of memory, restart R (_Session > Restart R_) after each lab. +::: + +```bash +cd /path/to/workshop-dir # replace this with the full path to the workshop folder +docker pull ghcr.io/NBISweden/workshop-scRNAseq:2023-bioconductor-r4.3.0 +docker run --rm -ti -p 8787:8787 -e PASSWORD=scrnaseq -v ${PWD}:/home/rstudio/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-bioconductor-r4.3.0 +``` + +In the browser, go to [localhost:8787](localhost:8787). Use the following credentials to log in to the RStudio Server: + +> User: `rstudio` +> Password: `scrnaseq` + +::: {.callout-important} +Do not close this terminal. +::: + + +### Scanpy + +::: {.callout-tip} +To avoid running out of memory, restart the kernel (_Kernel > Restart Kernel_) after each lab. +::: + +```bash +cd /path/to/workshop-dir # replace this with the full path to the workshop folder +docker pull ghcr.io/NBISweden/workshop-scRNAseq:2023-scanpy-py3.10 +docker run --rm -ti -p 8888:8888 -v ${PWD}:/home/jovyan/workdir ghcr.io/NBISweden/workshop-scRNAseq:2023-scanpy-py3.10 +``` + +At the end of the prompt, you will see a URL that starts with `http://127.0.0.1`, similar to the one below: + +``` +http://127.0.0.1:8888/lab?token=0a1d9ec51b91528a1d1fe2ad2c74f59ecb94c47070c2911d +``` + +Note that your token value will be different. Copy the entire URL (with the token) and paste it in your browser. + +::: {.callout-important} +Do not close this terminal. +::: diff --git a/containers/scripts/install_bioconductor.sh b/containers/scripts/install_bioconductor.sh new file mode 100755 index 00000000..4d13082a --- /dev/null +++ b/containers/scripts/install_bioconductor.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +set -e + +## Build ARGs +NCPUS=${NCPUS:--1} + +## Function to install apt packages only if they are not installed +function apt_install() { + if ! dpkg -s "$@" >/dev/null 2>&1; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + apt-get update + fi + apt-get install -y --no-install-recommends "$@" + fi +} + +apt_install \ + libbz2-dev \ + libglpk-dev \ + libxt-dev \ + libhdf5-dev \ + patch \ + python3.10-dev \ + python3-pip \ + python3.10-venv \ + vim + +## Install Miniconda +wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/rstudio/miniconda.sh +/bin/bash /home/rstudio/miniconda.sh -b -p /opt/conda +rm -rf /home/rstudio/miniconda.sh + +## Init conda for root and rstudio users +/opt/conda/bin/conda init bash +su - rstudio -c "/opt/conda/bin/conda init bash" + +## Create convenience symlink for installBioc.r +ln -sf \ + "${R_HOME}/site-library/littler/examples/installBioc.r" \ + /usr/local/bin/installBioc.r + +## Install R packages from CRAN +install2.r --error --skipinstalled -n "$NCPUS" \ + Seurat \ + hdf5r \ + umap + +## Install R packages with BiocManager (https://stackoverflow.com/a/62456026) +installBioc.r --error --skipinstalled -n "$NCPUS" \ + batchelor \ + biomaRt \ + clustree \ + enrichR \ + fgsea \ + igraph \ + msigdbr \ + org.Hs.eg.db \ + pheatmap \ + rafalib \ + scater \ + scDblFinder \ + scmap \ + scran \ + SingleR \ + slingshot \ + Spaniel \ + tradeSeq + +## Install R packages from GitHub +installGithub.r \ + https://github.com/powellgenomicslab/scPred/tree/af5492e778b076e529c20462c92aacd06c75bdc0 + +## Clean up +rm -rf /var/lib/apt/lists/* +rm -rf /tmp/downloaded_packages + +## Strip binary installed libraries from RSPM +## https://github.com/rocker-org/rocker-versioned2/issues/340 +strip /usr/local/lib/R/site-library/*/libs/*.so diff --git a/containers/scripts/install_bioconductor_spatial.sh b/containers/scripts/install_bioconductor_spatial.sh new file mode 100755 index 00000000..a75f0816 --- /dev/null +++ b/containers/scripts/install_bioconductor_spatial.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + +## Build ARGs +NCPUS=${NCPUS:--1} + +## Function to install apt packages only if they are not installed +function apt_install() { + if ! dpkg -s "$@" >/dev/null 2>&1; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + apt-get update + fi + apt-get install -y --no-install-recommends "$@" + fi +} + +apt_install \ + libbz2-dev \ + libglpk-dev \ + libxt-dev \ + libhdf5-dev \ + patch \ + vim + +## Install Miniconda +wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/rstudio/miniconda.sh +/bin/bash /home/rstudio/miniconda.sh -b -p /opt/conda +rm -rf /home/rstudio/miniconda.sh + +## Init conda for root and rstudio users +/opt/conda/bin/conda init bash +su - rstudio -c "/opt/conda/bin/conda init bash" + +## Create convenience symlink for installBioc.r +ln -sf \ + "${R_HOME}/site-library/littler/examples/installBioc.r" \ + /usr/local/bin/installBioc.r + +## Install packages with BiocManager (https://stackoverflow.com/a/62456026) +installBioc.r --error --skipinstalled -n "$NCPUS" \ + batchelor \ + biomaRt \ + scater \ + scran \ + SingleCellExperiment \ + SingleR \ + Spaniel + +## Clean up +rm -rf /var/lib/apt/lists/* +rm -rf /tmp/downloaded_packages + +## Strip binary installed libraries from RSPM +## https://github.com/rocker-org/rocker-versioned2/issues/340 +strip /usr/local/lib/R/site-library/*/libs/*.so diff --git a/containers/scripts/install_scanpy.sh b/containers/scripts/install_scanpy.sh new file mode 100755 index 00000000..d81d7eec --- /dev/null +++ b/containers/scripts/install_scanpy.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +## Build ARGs +NCPUS=${NCPUS:--1} +QUARTO_VERSION="1.3.450" + +## Function to install apt packages only if they are not installed +function apt_install() { + if ! dpkg -s "$@" >/dev/null 2>&1; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + apt-get update + fi + apt-get install -y --no-install-recommends "$@" + fi +} + +apt_install \ + curl \ + g++ \ + gcc \ + gdebi \ + libfmt-dev + +## Install quarto cli +curl -o quarto-linux-${TARGETARCH}.deb -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${TARGETARCH}.deb +gdebi --non-interactive quarto-linux-${TARGETARCH}.deb +rm -rf quarto-linux-${TARGETARCH}.deb + +## Install Quarto Jupyter extension +python3 -m pip install jupyterlab-quarto + +## Clean up +apt-get clean +rm -rf /var/lib/apt/lists/* +rm -rf /tmp/downloaded_packages diff --git a/containers/scripts/install_scanpy_spatial.sh b/containers/scripts/install_scanpy_spatial.sh new file mode 100755 index 00000000..d81d7eec --- /dev/null +++ b/containers/scripts/install_scanpy_spatial.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +## Build ARGs +NCPUS=${NCPUS:--1} +QUARTO_VERSION="1.3.450" + +## Function to install apt packages only if they are not installed +function apt_install() { + if ! dpkg -s "$@" >/dev/null 2>&1; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + apt-get update + fi + apt-get install -y --no-install-recommends "$@" + fi +} + +apt_install \ + curl \ + g++ \ + gcc \ + gdebi \ + libfmt-dev + +## Install quarto cli +curl -o quarto-linux-${TARGETARCH}.deb -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${TARGETARCH}.deb +gdebi --non-interactive quarto-linux-${TARGETARCH}.deb +rm -rf quarto-linux-${TARGETARCH}.deb + +## Install Quarto Jupyter extension +python3 -m pip install jupyterlab-quarto + +## Clean up +apt-get clean +rm -rf /var/lib/apt/lists/* +rm -rf /tmp/downloaded_packages diff --git a/containers/scripts/install_seurat.sh b/containers/scripts/install_seurat.sh new file mode 100755 index 00000000..9ca006dc --- /dev/null +++ b/containers/scripts/install_seurat.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -e + +## Build ARGs +NCPUS=${NCPUS:--1} + +## Function to install apt packages only if they are not installed +function apt_install() { + if ! dpkg -s "$@" >/dev/null 2>&1; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + apt-get update + fi + apt-get install -y --no-install-recommends "$@" + fi +} + +apt_install \ + libhdf5-dev \ + libglpk-dev \ + patch \ + python3.10-dev \ + python3-pip \ + python3.10-venv \ + vim + +## Install Miniconda +wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/rstudio/miniconda.sh +/bin/bash /home/rstudio/miniconda.sh -b -p /opt/conda +rm -rf /home/rstudio/miniconda.sh + +## Init conda for root and rstudio users +/opt/conda/bin/conda init bash +su - rstudio -c "/opt/conda/bin/conda init bash" + +## Create convenience symlink for installBioc.r +ln -sf \ + "${R_HOME}/site-library/littler/examples/installBioc.r" \ + /usr/local/bin/installBioc.r + +## Install R packages from CRAN +install2.r --error --skipinstalled -n "$NCPUS" \ + Seurat \ + hdf5r \ + umap + +## Install R packages with BiocManager (https://stackoverflow.com/a/62456026) +installBioc.r --error --skipinstalled -n "$NCPUS" \ + biomaRt \ + clustree \ + edgeR \ + enrichR \ + fgsea \ + lme4 \ + MAST \ + msigdbr \ + pheatmap \ + rafalib \ + scran \ + slingshot \ + tradeSeq + +## Install R packages from GitHub +installGithub.r \ + https://github.com/chris-mcginnis-ucsf/DoubletFinder/tree/1b1d4e2d7f893a3552d9f8f791ab868ee4c782e6 \ + https://github.com/immunogenomics/harmony/tree/f054b030b5d503e7c385dac6290604013ab9f81b \ + https://github.com/powellgenomicslab/scPred/tree/af5492e778b076e529c20462c92aacd06c75bdc0 + +## Clean up +rm -rf /var/lib/apt/lists/* +rm -rf /tmp/downloaded_packages + +## Strip binary installed libraries from RSPM +## https://github.com/rocker-org/rocker-versioned2/issues/340 +strip /usr/local/lib/R/site-library/*/libs/*.so diff --git a/containers/scripts/install_seurat_spatial.sh b/containers/scripts/install_seurat_spatial.sh new file mode 100755 index 00000000..374dc0fc --- /dev/null +++ b/containers/scripts/install_seurat_spatial.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +## Build ARGs +NCPUS=${NCPUS:--1} + +## Function to install apt packages only if they are not installed +function apt_install() { + if ! dpkg -s "$@" >/dev/null 2>&1; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + apt-get update + fi + apt-get install -y --no-install-recommends "$@" + fi +} + +apt_install \ + libhdf5-dev \ + libglpk-dev \ + patch \ + vim + +## Install Miniconda +wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /home/rstudio/miniconda.sh +/bin/bash /home/rstudio/miniconda.sh -b -p /opt/conda +rm -rf /home/rstudio/miniconda.sh + +## Init conda for root and rstudio users +/opt/conda/bin/conda init bash +su - rstudio -c "/opt/conda/bin/conda init bash" + +## Create convenience symlink for installBioc.r +ln -sf \ + "${R_HOME}/site-library/littler/examples/installBioc.r" \ + /usr/local/bin/installBioc.r + +## Install packages from CRAN +install2.r --error --skipinstalled -n "$NCPUS" \ + hdf5r \ + umap \ + Seurat + +## Install packages with BiocManager (https://stackoverflow.com/a/62456026) +installBioc.r --error --skipinstalled -n "$NCPUS" \ + Biobase + +## Install packages from GitHub +installGithub.r \ + https://github.com/renozao/xbioc/tree/1354168bd7e64be4ee1f9f74e971a61556d75003 \ + https://github.com/meichendong/SCDC/tree/890c604eebd7fffa4a08d7344fbd516df6efcf8d \ + https://github.com/satijalab/seurat-data/tree/dd7d0df1fe7cd563281eac91edd0f1fac1ff3fe5 + +## Clean up +rm -rf /var/lib/apt/lists/* +rm -rf /tmp/downloaded_packages + +## Strip binary installed libraries from RSPM +## https://github.com/rocker-org/rocker-versioned2/issues/340 +strip /usr/local/lib/R/site-library/*/libs/*.so diff --git a/containers/scripts/reticulate_install_bioconductor.R b/containers/scripts/reticulate_install_bioconductor.R new file mode 100755 index 00000000..03ef608d --- /dev/null +++ b/containers/scripts/reticulate_install_bioconductor.R @@ -0,0 +1,12 @@ +library(reticulate) + +conda_create( + envname = "bioc", + python_version = "3.8", + channel = c("conda-forge", "bioconda", "anaconda"), + conda = "/opt/conda/bin/conda" +) + +virtualenv_create("/opt/venv/scanorama") +use_virtualenv("/opt/venv/scanorama") +py_install("scanorama==1.7.3") diff --git a/containers/scripts/reticulate_install_bioconductor_spatial.R b/containers/scripts/reticulate_install_bioconductor_spatial.R new file mode 100755 index 00000000..ce6a430d --- /dev/null +++ b/containers/scripts/reticulate_install_bioconductor_spatial.R @@ -0,0 +1,8 @@ +library(reticulate) + +conda_create( + envname = "bioc-spatial", + python_version = "3.8", + channel = c("conda-forge", "bioconda", "anaconda"), + conda = "/opt/conda/bin/conda" +) \ No newline at end of file diff --git a/containers/scripts/reticulate_install_seurat.R b/containers/scripts/reticulate_install_seurat.R new file mode 100755 index 00000000..26480454 --- /dev/null +++ b/containers/scripts/reticulate_install_seurat.R @@ -0,0 +1,13 @@ +library(reticulate) + +conda_create( + envname = "seurat", + python_version = "3.8", + packages = c("umap-learn==0.5.4"), + channel = c("conda-forge", "bioconda", "anaconda"), + conda = "/opt/conda/bin/conda" +) + +virtualenv_create("/opt/venv/scanorama") +use_virtualenv("/opt/venv/scanorama") +py_install("scanorama==1.7.3") diff --git a/containers/scripts/reticulate_install_seurat_spatial.R b/containers/scripts/reticulate_install_seurat_spatial.R new file mode 100755 index 00000000..229c35e7 --- /dev/null +++ b/containers/scripts/reticulate_install_seurat_spatial.R @@ -0,0 +1,8 @@ +library(reticulate) + +conda_create( + envname = "seurat-spatial", + python_version = "3.10", + channel = c("conda-forge", "bioconda", "anaconda"), + conda = "/opt/conda/bin/conda" +) diff --git a/containers/uppmax/launch_jupyter.sh b/containers/uppmax/launch_jupyter.sh new file mode 100755 index 00000000..7ef40f67 --- /dev/null +++ b/containers/uppmax/launch_jupyter.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +SIF=$1 + +export APPTAINERENV_USER=$(id -un) + +## Get unused socket per https://unix.stackexchange.com/a/132524 +## Tiny race condition between the python & singularity commands +readonly PORT=$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') +cat 1>&2 <&2 diff --git a/containers/uppmax/launch_rstudio.sh b/containers/uppmax/launch_rstudio.sh new file mode 100755 index 00000000..74b4f9c6 --- /dev/null +++ b/containers/uppmax/launch_rstudio.sh @@ -0,0 +1,84 @@ +#!/bin/sh + +SIF=$1 + +## Create temporary directory to be populated with directories to bind-mount in the container +## where writable file systems are necessary. Adjust path as appropriate for your computing environment. +workdir=$(python -c 'import tempfile; print(tempfile.mkdtemp())') + +echo "container workdir: ${workdir}" + +mkdir -p -m 700 ${workdir}/run ${workdir}/tmp ${workdir}/var/lib/rstudio-server +cat > ${workdir}/database.conf < ${workdir}/rsession.sh <&2 <&2 diff --git a/containers/uppmax/singularity_bioconductor.def b/containers/uppmax/singularity_bioconductor.def new file mode 100644 index 00000000..d1633366 --- /dev/null +++ b/containers/uppmax/singularity_bioconductor.def @@ -0,0 +1,7 @@ +Bootstrap: docker +From: ghcr.io/NBISweden/workshop-scRNAseq:2023-bioconductor-r4.3.0 + +%post +chmod -R 777 /opt/conda +chmod -R 777 /opt/venv + diff --git a/containers/uppmax/singularity_scanpy.def b/containers/uppmax/singularity_scanpy.def new file mode 100644 index 00000000..a5068acf --- /dev/null +++ b/containers/uppmax/singularity_scanpy.def @@ -0,0 +1,5 @@ +Bootstrap: docker +From: ghcr.io/NBISweden/workshop-scRNAseq:2023-scanpy-py3.10 + +%post +chmod -R 777 /opt/conda diff --git a/containers/uppmax/singularity_seurat.def b/containers/uppmax/singularity_seurat.def new file mode 100644 index 00000000..b313392e --- /dev/null +++ b/containers/uppmax/singularity_seurat.def @@ -0,0 +1,7 @@ +Bootstrap: docker +From: ghcr.io/NBISweden/workshop-scRNAseq:2023-seurat-r4.3.0 + +%post +chmod -R 777 /opt/conda +chmod -R 777 /opt/venv +