Replace MinIO / S3 with cached WhiteNoise and FileSystemStorage

.dockerignore

@@ -1,15 +1,15 @@
 .DS_Store
+.git
+Dockerfile
+.dockerignore
 
 # Python
 *.pyc
 __pycache__
 /venv
 /.venv
 .coverage
-
-# Compiled Static Files
-# Enabled again for compiling static files in multiple steps
-# static/
+.pytest_cache
 
 # Node
 /node_modules
@@ -21,15 +21,26 @@ __pycache__
 # IDE
 .idea
 .vscode
-README.md
+
+# Other development files
 requirements-dev.txt
 artwork
 .github
 fixture.json
+pyproject.toml
+.eslintrc.js
+README.md
+docker-compose.yml
+.pre-commit-config.yaml
+.readthedocs.yaml
+environment.yml
+
 
-# Generated PDF files
+# Generated files
 /*.pdf
 /media
+/static
+/gcampus/*/static
 
-# Generated documentation
-/docs/_build
+# Documentation
+/docs

.github/workflows/release.yml

@@ -12,26 +12,29 @@ jobs:
   build:
     if: '!github.event.release.draft'
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: ghcr.io/${{ github.repository }}
           # In the code below, we enable the 'dev' or 'latest' tag depending
@@ -43,12 +46,15 @@ jobs:
             latest=${{ !github.event.release.prerelease }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: true
+          platforms: 'linux/amd64,linux/arm64'
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:latest
+          cache-to: type=inline
 
     outputs:
       tag: ${{ steps.meta.outputs.version }}
@@ -58,12 +64,14 @@ jobs:
     if: '!github.event.release.draft'
     runs-on: ubuntu-latest
     needs: build
+    permissions:
+      contents: read
     env:
       current: 'ghcr.io/${{ github.repository }}:${{ needs.build.outputs.tag }}'
 
     steps:
       - name: Log in to the Container registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -94,54 +102,3 @@ jobs:
         run: |
           docker service update gcampus_gcampus_dev --with-registry-auth --image="${{ env.current }}"
           docker service update gcampus_gcampus_worker_dev --with-registry-auth --image="${{ env.current }}"
-
-  static:
-    if: '!github.event.release.draft'
-    runs-on: ubuntu-latest
-    needs: build
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install dependencies with apt
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends --no-install-suggests pandoc binutils libproj-dev gdal-bin libpango-1.0-0 libpangoft2-1.0-0
-
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-          cache: 'npm'
-
-      - name: Install Node.js packages
-        run: npm ci --also=dev
-
-      - name: Build static files
-        run: npm run build
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-          cache: 'pip'
-
-      - name: Install Requirements
-        run: pip install -r requirements.txt
-
-      - name: Upload static files (production environment)
-        if: '!github.event.release.prerelease'
-        run: python manage.py collectstatic --no-input
-        env:
-          USE_S3_STORAGE: True
-          S3_ACCESS_KEY: ${{secrets.MINIO_ACCESS_KEY}}
-          S3_SECRET_KEY: ${{secrets.MINIO_SECRET_KEY}}
-          S3_ENDPOINT_URL: 'https://files.gewaessercampus.de'
-          S3_BUCKET_NAME: 'gcampus'
-
-      - name: Upload static files (staging environment)
-        run: python manage.py collectstatic --no-input
-        env:
-          USE_S3_STORAGE: True
-          S3_ACCESS_KEY: ${{secrets.MINIO_ACCESS_KEY}}
-          S3_SECRET_KEY: ${{secrets.MINIO_SECRET_KEY}}
-          S3_ENDPOINT_URL: 'https://files.gewaessercampus.de'
-          S3_BUCKET_NAME: 'gcampus-dev'

.github/workflows/test.yml

@@ -9,12 +9,14 @@ concurrency:
 jobs:
   test:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     strategy:
       fail-fast: false
       max-parallel: 4
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
-    
+        python-version: ['3.10', '3.11']
+
     env:
       GCAMPUS_DB_USER: 'gcampus'
       GCAMPUS_DB_NAME: 'gcampustest'
@@ -49,11 +51,11 @@ jobs:
       - name: Install dependencies with apt
         run: |
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends --no-install-suggests pandoc binutils libproj-dev gdal-bin libpango-1.0-0 libpangoft2-1.0-0
+          sudo apt-get install -y --no-install-recommends --no-install-suggests binutils libproj-dev gdal-bin libpango-1.0-0 libpangoft2-1.0-0
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: '20'
           cache: 'npm'
 
       - name: Install Node.js packages
@@ -70,6 +72,12 @@ jobs:
       - name: Install Requirements
         run: pip install -r requirements.txt
 
+      - name: Collect static files
+        run: python manage.py collectstatic --no-input
+        env:
+          DJANGO_SETTINGS_MODULE: 'gcampus.settings.test'
+          GCAMPUS_REDIS_HOST: 'localhost'
+
       - name: Run Tests
         run: python manage.py test
         env:

.pre-commit-config.yaml

@@ -2,10 +2,10 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.3.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort

Dockerfile

@@ -1,54 +1,64 @@
 ########################################################################
 # Static files
 ########################################################################
-FROM node:16 AS static
+FROM node:20 AS static
 LABEL maintainer="Jonas Drotleff <j.drotleff@desk-lab.de>"
 
 WORKDIR /usr/src/gcampus
 COPY package*.json ./
-RUN npm ci && npm rebuild node-sass --unsafe-perm
+RUN npm ci --also=dev
 
 COPY . .
 RUN npm run build && rm -rf gcampus/*/static_src package*.json node_modules
 
 ########################################################################
 # gcampus Dockerfile
 ########################################################################
-FROM python:3.10-slim
+FROM python:3.11-slim
 LABEL maintainer="Jonas Drotleff <j.drotleff@desk-lab.de>"
 
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/sh", "-eux", "-c"]
+
+ENV VIRTUAL_ENV=/home/gcampus/venv
 ENV DJANGO_SETTINGS_MODULE gcampus.settings.prod
+ENV DEBIAN_FRONTEND=noninteractive
 USER root
 
 ########################################################################
 # Setup GCampus
 ########################################################################
 
 # Install gdal and libproj-dev
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends --no-install-suggests binutils pandoc libproj-dev gdal-bin python3-pip python3-gi libpango-1.0-0 libpangoft2-1.0-0 && \
-    rm -rf /var/lib/apt/lists/*
-
-# Create gcampus directory and user
-RUN useradd -m gcampus && \
-    mkdir  -p /srv/gcampus && \
-    chown -R gcampus /srv/gcampus
+RUN apt-get update && apt-get -y upgrade && \
+    apt-get install -y --no-install-recommends --no-install-suggests \
+    binutils \
+    libgdal28 \
+    libpango-1.0-0 \
+    libpangoft2-1.0-0 \
+    libproj-dev && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /var/log
 
-WORKDIR /srv/gcampus
+# Create gcampus user
+RUN useradd --create-home gcampus
 USER gcampus
+WORKDIR /home/gcampus
 
-COPY --from=static --chown=gcampus /usr/src/gcampus/requirements.txt /srv/gcampus/requirements.txt
+# Copy requirements.txt file
+COPY --from=static --chown=gcampus /usr/src/gcampus/requirements.txt ./requirements.txt
 
-# Install dependencies for gcampus
-RUN python3 -m venv venv && \
-    /srv/gcampus/venv/bin/pip install --no-cache-dir -r requirements.txt
+# Create venv for gcampus
+RUN python3 -m venv ${VIRTUAL_ENV}
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-COPY --from=static --chown=gcampus /usr/src/gcampus /srv/gcampus
+# Install requirements
+RUN pip install --disable-pip-version-check --no-cache-dir -r requirements.txt
 
-RUN chmod +x /srv/gcampus/docker-entrypoint.sh
+COPY --from=static --chown=gcampus /usr/src/gcampus ./
+
+RUN GCAMPUS_ALLOWED_HOSTS="" python manage.py collectstatic --no-input && rm -rf gcampus/*/static
+
+RUN chmod +x ./docker-entrypoint.sh
 
-USER gcampus
 EXPOSE 8000
 ENTRYPOINT ["./docker-entrypoint.sh"]

docker-entrypoint.sh

@@ -20,7 +20,7 @@ set -eo pipefail
 shopt -s nullglob
 
 # Migrate database
-/srv/gcampus/venv/bin/python3 manage.py migrate
+python3 manage.py migrate
 
 # Launch gunicorn server
-/srv/gcampus/venv/bin/gunicorn
+gunicorn

docs/development/setup.md

@@ -9,7 +9,6 @@ this requirements serve a specific purpose as listed below:
  - **Web framework**: [Django](https://www.djangoproject.com) (with the [GeoDjango module](https://docs.djangoproject.com/en/4.0/ref/contrib/gis/))
  - **Database**: [PostgreSQL](https://www.postgresql.org) (with the [PostGIS spatial database extender](https://postgis.net))
  - **Task queue**: [Celery](https://docs.celeryproject.org/en/stable/index.html) with [Redis](https://redis.io)
- - **Emails**: [pandoc](https://pandoc.org) to turn HTML emails into a plaintext alternative
  - **Document factory**: [WeasyPrint](https://weasyprint.org) (used to create downloadable PDFs)
  - **JavaScript bundler**: [webpack](https://webpack.js.org)
 
@@ -39,7 +38,6 @@ tells conda to install all the required packages such as:
  - `libgdal`
  - `glib`
  - `pango`
- - `pandoc`
 
 All other Python dependencies located in the `requirements.txt` will be
 installed automatically using `pip`. This is also where the Celery

environment.yml

@@ -3,11 +3,10 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.10
-  - pip=22
+  - python=3.11
+  - pip=23
   - libgdal=3.4
   - glib=2.72
   - pango=1.50
-  - pandoc=2.19
   - pip:
       - '-r requirements-dev.txt'

gcampus/__init__.py

@@ -15,5 +15,5 @@
 
 from django.utils.version import get_version
 
-VERSION = (0, 7, 2, "final", 0)
+VERSION = (0, 8, 0, "rc", 2)
 __version__ = get_version(VERSION)

gcampus/auth/apps.py

@@ -26,6 +26,5 @@ class GCampusAuthAppConfig(AppConfig):
 
     def ready(self):
         from . import receivers  # imported to connect all receivers
-        from .models import update_last_token_login
 
-        token_user_logged_in.connect(update_last_token_login)
+        super().ready()

gcampus/auth/context_processors.py

@@ -33,5 +33,4 @@ def auth(request: HttpRequest) -> dict:
         "user_token_is_course_token": (
             session.get_token_type(request) is TokenType.course_token
         ),
-        "email_confirmation_timeout": settings.EMAIL_CONFIRMATION_TIMEOUT.days,
     }

gcampus/auth/models/__init__.py

@@ -13,13 +13,6 @@
 #  You should have received a copy of the GNU Affero General Public License
 #  along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from django.utils import timezone
-
 from gcampus.auth.models.course import Course
 from gcampus.auth.models.token import BaseToken, AccessKey, CourseToken
 from gcampus.auth.models.user import User
-
-
-def update_last_token_login(sender, instance: BaseToken, **kwargs):  # noqa
-    instance.last_login = timezone.now()
-    instance.save(update_fields=["last_login"])

gcampus/auth/models/course.py

@@ -114,7 +114,7 @@ def check_token(self, course, token):
 
         # Check the timestamp is within limit.
         token_age = timedelta(seconds=(self._num_seconds(self._now()) - ts))
-        if token_age > settings.EMAIL_CONFIRMATION_TIMEOUT:
+        if token_age > settings.CONFIRMATION_TIMEOUT:
             raise EmailVerificationExpired()
 
         return True