From c3e7705ea96b8c10524323ca6680fcf4a3898a85 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Mar 2024 14:44:17 +0100 Subject: [PATCH 01/22] Add version streamflow --- .github/workflows/build_job.yml | 6 +++++- Dockerfile | 11 +++++++++-- package.json | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index f737369f..74a70163 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -7,7 +7,7 @@ run-name: ${{ github.actor }} is building the container tags: - '*' branches: - - 'main' + - 'version' jobs: build-and-push-image: @@ -33,6 +33,10 @@ jobs: name: Set up QEMU uses: docker/setup-qemu-action@v1 + - + name: Set docker version label + run: sed -i 's//${{ steps.repository.outputs.repo }}/g' + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 diff --git a/Dockerfile b/Dockerfile index 5d664ee0..566fc884 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG FREVA_WEB_DIR=/opt/freva_web FROM condaforge/mambaforge LABEL maintainer="DRKZ-CLINT" LABEL repository="https://github.com/FREVA-CLINT/freva-web" - +LABEL release="" ARG CONDA_ENV_DIR ARG FREVA_WEB_DIR @@ -13,14 +13,21 @@ COPY . . ENV PATH=$CONDA_ENV_DIR/bin:$PATH\ DJANGO_SUPERUSER_EMAIL=freva@dkrz.de +RUN set -e && \ + groupadd -r --gid 1000 freva && \ + adduser --uid "freva" --gid 1000 --gecos "Freva user" \ + --shell /bin/bash --disabled-password "freva" --home ${FREVA_WEB_DIR} && \ + chown -R freva:freva ${FREVA_WEB_DIR} + RUN set -e \ && mamba env create -p ${CONDA_ENV_DIR} -f conda-env.yml \ && npm install && npm run build-production \ && mamba clean -afy \ && rm -rf node_modules \ - && echo "export PATH=${PATH}" >> /root/.bashrc + && echo "export PATH=${PATH}" >> ${FREVA_WEB_DIR}/.bashrc EXPOSE 8000 +USER freva CMD ./init_django.sh diff --git a/package.json b/package.json index 641f8c72..f021061b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "evaluation_system_web", - "version": "2023.08.23", + "version": "2403.0.0", "description": "React-bits of the freva-web interface. The react-parts of the web interface include the plugin-selection, the data-browser and the result-browser", "main": "index.js", "engines": { From f0020cc0ff8babe33993395d89dbdf2698a582e4 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Mar 2024 14:47:05 +0100 Subject: [PATCH 02/22] Fix typos --- .github/workflows/build_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 74a70163..0b33bbbc 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -35,7 +35,7 @@ jobs: - name: Set docker version label - run: sed -i 's//${{ steps.repository.outputs.repo }}/g' + run: sed -i 's//${{ steps.repository.outputs.tag }}/g' - name: Set up Docker Buildx From d0992674d3c6474261f916381672bb2a59603953 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Mar 2024 14:52:01 +0100 Subject: [PATCH 03/22] Fix typo --- .github/workflows/build_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 0b33bbbc..3bc01a3b 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -35,7 +35,7 @@ jobs: - name: Set docker version label - run: sed -i 's//${{ steps.repository.outputs.tag }}/g' + run: sed -i Dockerfile 's//${{ steps.repository.outputs.tag }}/g' - name: Set up Docker Buildx From 3b52867821b900da88c0387672f38f107ad91570 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Mar 2024 14:53:49 +0100 Subject: [PATCH 04/22] Fix linting issues. --- .ci/install_conda.py | 1 + django_evaluation/monitor.py | 1 + django_evaluation/wsgi.py | 1 + solr/views.py | 1 + 4 files changed, 4 insertions(+) diff --git a/.ci/install_conda.py b/.ci/install_conda.py index 04580c82..6f26c814 100755 --- a/.ci/install_conda.py +++ b/.ci/install_conda.py @@ -1,4 +1,5 @@ """Deploy the evaluation_system / core.""" + import argparse import logging from pathlib import Path diff --git a/django_evaluation/monitor.py b/django_evaluation/monitor.py index ccb0629a..5f3a052f 100644 --- a/django_evaluation/monitor.py +++ b/django_evaluation/monitor.py @@ -2,6 +2,7 @@ This is an example code for restarting after code changes from http://code.google.com/p/modwsgi/wiki/ReloadingSourceCode#Restarting_Daemon_Processes """ + import os import sys import signal diff --git a/django_evaluation/wsgi.py b/django_evaluation/wsgi.py index 95db6523..268786cc 100644 --- a/django_evaluation/wsgi.py +++ b/django_evaluation/wsgi.py @@ -13,6 +13,7 @@ framework. """ + import os import sys import site diff --git a/solr/views.py b/solr/views.py index 48cbd92f..4910c84e 100644 --- a/solr/views.py +++ b/solr/views.py @@ -5,6 +5,7 @@ views for the solr application """ + from typing import Union from django.shortcuts import render from django.contrib.auth.decorators import login_required From 93d2ea4bebd9ba2bfc41be166bdc51e42b90589c Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Mar 2024 14:57:53 +0100 Subject: [PATCH 05/22] Fix typo --- .github/workflows/build_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 3bc01a3b..a70c7c83 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -35,7 +35,7 @@ jobs: - name: Set docker version label - run: sed -i Dockerfile 's//${{ steps.repository.outputs.tag }}/g' + run: sed -i 's//${{ steps.repository.outputs.tag }}/g' Dockerfile - name: Set up Docker Buildx From 04b148b51dd3594a1b98bd34333421f4092b93a6 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Mar 2024 22:10:10 +0100 Subject: [PATCH 06/22] fix docker build issues. --- Dockerfile | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 566fc884..e597ab7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,30 +4,26 @@ ARG FREVA_WEB_DIR=/opt/freva_web FROM condaforge/mambaforge LABEL maintainer="DRKZ-CLINT" LABEL repository="https://github.com/FREVA-CLINT/freva-web" -LABEL release="" +LABEL release= ARG CONDA_ENV_DIR ARG FREVA_WEB_DIR +RUN set -e && \ + groupadd -r -g 200855 freva && \ + adduser --uid 1000 --gid 200855 --gecos "Freva user" \ + --shell /bin/bash --disabled-password freva --home ${FREVA_WEB_DIR} &&\ + mkdir -p ${CONDA_ENV_DIR} && chown -R freva:freva $CONDA_ENV_DIR WORKDIR ${FREVA_WEB_DIR} COPY . . - ENV PATH=$CONDA_ENV_DIR/bin:$PATH\ DJANGO_SUPERUSER_EMAIL=freva@dkrz.de -RUN set -e && \ - groupadd -r --gid 1000 freva && \ - adduser --uid "freva" --gid 1000 --gecos "Freva user" \ - --shell /bin/bash --disabled-password "freva" --home ${FREVA_WEB_DIR} && \ - chown -R freva:freva ${FREVA_WEB_DIR} - - -RUN set -e \ - && mamba env create -p ${CONDA_ENV_DIR} -f conda-env.yml \ - && npm install && npm run build-production \ - && mamba clean -afy \ - && rm -rf node_modules \ - && echo "export PATH=${PATH}" >> ${FREVA_WEB_DIR}/.bashrc - -EXPOSE 8000 +RUN set -e && \ + mamba env create -y -p ${CONDA_ENV_DIR} -f conda-env.yml &&\ + mamba clean -afy &&\ + npm install && npm run build-production &&\ + rm -rf node_modules &&\ + echo "export PATH=${PATH}" >> ${FREVA_WEB_DIR}/.bashrc &&\ + chown -R freva:freva ${FREVA_WEB_DIR} USER freva - +EXPOSE 8000 CMD ./init_django.sh From aa09e1df259f9a14f768e634d010d26b9e64866e Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Sat, 16 Mar 2024 00:16:26 +0100 Subject: [PATCH 07/22] Add better version tags. --- .github/workflows/build_job.yml | 2 +- Dockerfile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index a70c7c83..4556288c 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -26,7 +26,7 @@ jobs: name: Get tag id: repository run: | - echo "tag=v$(cat package.json|jq -r .version)" >> $GITHUB_OUTPUT + echo "tag=$(cat package.json|jq -r .version)" >> $GITHUB_OUTPUT echo "repo=$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')" >> $GITHUB_OUTPUT - diff --git a/Dockerfile b/Dockerfile index e597ab7f..a06c1d39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,9 @@ ARG CONDA_ENV_DIR=/opt/condaenv ARG FREVA_WEB_DIR=/opt/freva_web FROM condaforge/mambaforge -LABEL maintainer="DRKZ-CLINT" -LABEL repository="https://github.com/FREVA-CLINT/freva-web" -LABEL release= +LABEL org.opencontainers.image.authors="DRKZ-CLINT" +LABEL org.opencontainers.image.source="https://github.com/FREVA-CLINT/freva-web" +LABEL org.opencontainers.image.version= ARG CONDA_ENV_DIR ARG FREVA_WEB_DIR From 0f3a690cfa16cb2a147c2078c9de844fd633013e Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 18 Mar 2024 01:48:19 +0100 Subject: [PATCH 08/22] Do not use tox. --- CHANGELOG.md | 23 ++++++ Makefile | 4 + README.md | 18 ++--- release.py | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 257 insertions(+), 12 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 release.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..fb307566 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +All notable changes to this project will be documented in this file. +## [v2403.0.0] + +### Added +- Change lock file, to keep track of changes. + + +# Template: +## [Unreleased] + +### Added +- New feature X. +- New feature Y. + +### Changed +- Improved performance in component A. +- Updated dependency B to version 2.0.0. + +### Fixed +- Fixed issue causing application crash on startup. +- Fixed bug preventing users from logging in. diff --git a/Makefile b/Makefile index e8c6c0d4..fcc52a9e 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,7 @@ lint: tests: rm -rf node_modules pytest -vv $(PWD) tests/ + +release: + pip install git-python packaging tomli + python3 release.py tag django_evaluation -b version diff --git a/README.md b/README.md index 4142d96e..d8be35e9 100644 --- a/README.md +++ b/README.md @@ -120,17 +120,11 @@ it to the registry. To do so please follow the following steps. "version": "2023.07.19" ``` - After you have push the version changes to the main branch you can create - a new tag with the same version: + trigger the release procedure: ```console - git tag -a vVERSION -m "comment" + make release ``` - for example: - ```console - git tag -a v2023.07.19 -m "Some prettifications." - ``` -- Push the tag to the remote repository: - ```console - git push origin vVERSION - ``` - -These steps trigger the creation of a new container image. +This will check the current version of the `main` branch and created a trigger +a GitHub continuous integration pipeline to create a new release. The procedure +performs a couple of checks, if theses checks fail please make sure to address +the issues. diff --git a/release.py b/release.py new file mode 100644 index 00000000..04a5cb8a --- /dev/null +++ b/release.py @@ -0,0 +1,224 @@ +"""Script that prepares a new release of a version.""" + +import argparse +import json +import logging +import re +import tempfile +from functools import cached_property +from itertools import product +from pathlib import Path + +import git +import tomli +from packaging.version import Version, InvalidVersion + +# Set up logging +logging.basicConfig( + format="%(name)s - %(levelname)s - %(message)s", + datefmt="[%X]", + level=logging.INFO, +) + + +logger = logging.getLogger("create-release") + + +class Exit(Exception): + """Custom error class for representing specific errors. + + Attributes: + message (str): A human-readable message describing the error. + code (int): An error code associated with the error. + """ + + def __init__(self, message: str, code: int = 1) -> None: + """Initialize the CustomError instance with the given message and error code. + + Args: + message (str): A human-readable message describing the error. + code (int): An error code associated with the error. + """ + super().__init__(message) + self.message = message + logger.critical(message) + raise SystemExit(code) + + +class Release: + """Release class.""" + + version_pattern: str = r'__version__\s*=\s*["\'](\d+\.\d+\.\d+)["\']' + + def __init__(self, package_name: str, repo_dir: str, branch: str = "main") -> None: + self.branch = branch + self.package_name = package_name + self.repo_dir = Path(repo_dir) + logger.info("Searching for packages/config with the name: %s", package_name) + logger.debug("Reading current git config") + self.git_config = ( + Path(git.Repo(search_parent_directories=True).git_dir) / "config" + ).read_text() + + def tag_version(self) -> None: + """Tag the latest git version.""" + + @cached_property + def repo_url(self) -> str: + """Get the current git repo url.""" + repo = git.Repo(search_parent_directories=True) + return repo.remotes.origin.url + + @cached_property + def git_tag(self) -> Version: + """Get the latest git tag.""" + logger.debug("Searching for the latest tag") + repo = git.Repo(self.repo_dir) + try: + # Get the latest tag on the main branch + return Version( + repo.git.describe("--tags", "--abbrev=0", self.branch).lstrip("v") + ) + except git.exc.GitCommandError: + logger.debug("No tag found") + except InvalidVersion: + logger.debug("Tag found, but could not parse version") + return Version("0.0.0") + + @property + def version(self) -> Version: + """Get the version of the current software.""" + logger.debug("Searching for software version.") + pck_dirs = Path("src") / self.package_name, Path( + "src" + ) / self.package_name.replace("-", "_") + files = [ + self.repo_dir / f[1] / f[0] + for f in product(("_version.py", "__init__.py"), pck_dirs) + ] + files += [ + self.repo_dir / Path("package.json"), + self.repo_dir / "pyproject.toml", + ] + for file in files: + if file.is_file(): + if file.suffix == ".py": + match = re.search(self.version_pattern, file.read_text()) + if match: + return Version(match.group(1)) + elif file.suffix == ".json": + content = json.loads(file.read_text()) + if "version" in content: + return Version(content["version"]) + elif file.suffix == ".toml": + content = tomli.loads(file.read_text()) + if "project" in content: + return Version(content["project"]["version"]) + raise ValueError("Could not find version") + + def _clone_repo_from_franch(self, branch: str = "main") -> None: + logger.debug( + "Cloning repository from %s with branch %s to %s", + self.repo_url, + self.repo_dir, + branch, + ) + git.Repo.clone_from(self.repo_url, self.repo_dir, branch=branch) + (self.repo_dir / ".git" / "config").write_text(self.git_config) + + def _check_change_lock_file(self) -> None: + """Check if the current version was added to the change lock file.""" + logger.debug("Checking for change log file.") + if not self._change_lock_file.is_file(): + raise Exit( + "Could not find change log file. " + f"Create one first and push it to the {self.branch} branch." + ) + if f"v{self.version}" not in self._change_lock_file.read_text("utf-8"): + raise Exit( + "You need to add the version v{} to the {} change log file " + "and push the update to the {} branch".format( + self.version, + self._change_lock_file.relative_to(self.repo_dir), + self.branch, + ) + ) + + @cached_property + def _change_lock_file(self) -> Path: + """Find the change lock file.""" + for prefix, suffix in product(("changelog", "whats-new"), (".rst", ".md")): + for search_pattern in (prefix, prefix.upper()): + glob_pattern = f"{search_pattern}{suffix}" + logger.debug("Searching for %s", glob_pattern) + for file in self.repo_dir.rglob(glob_pattern): + return file + return Path(tempfile.mktemp()) + + def tag_new_version(self) -> None: + """Tag a new git version.""" + self._clone_repo_from_franch(self.branch) + cloned_repo = git.Repo(self.repo_dir) + if self.version <= self.git_tag: + raise Exit( + "Tag version: {} is the same as current version {}" + ", you need to bump the verion number first and " + "push the changes to the {} branch".format( + self.version, + self.git_tag, + self.branch, + ) + ) + self._check_change_lock_file() + logger.info("Creating tag for version v%s", self.version) + head = cloned_repo.head.reference + message = f"Create a release for v{self.version}" + try: + cloned_repo.create_tag(f"v{self.version}", ref=head, message=message) + cloned_repo.git.push("--tags") + except git.GitCommandError as error: + raise Exit("Could not create tag: {}".format(error)) + logger.info("Tags created.") + + @classmethod + def cli(cls, temp_dir: str) -> "Release": + """Command line interface.""" + + parser = argparse.ArgumentParser( + description="Prepare the release of a package." + ) + subparser = parser.add_subparsers(help="Available commands:") + tag_parser = subparser.add_parser("tag", help="Create a new tag") + deploy_parser = subparser.add_parser( + "deploy", help="Update the version in the deployment repository" + ) + for _parser in tag_parser, deploy_parser: + _parser.add_argument("name", help="The name of the software/package.") + _parser.add_argument( + "-v", + "--verbose", + help="Enable debug mode.", + action="store_true", + ) + tag_parser.add_argument( + "-b", + "--branch", + help="Set the working branch", + type=str, + default="main", + ) + args = parser.parse_args() + if args.verbose: + logger.setLevel(logging.DEBUG) + return cls(args.name, temp_dir, args.branch) + + +if __name__ == "__main__": + with tempfile.TemporaryDirectory() as temporary_dir: + try: + release = Release.cli(temporary_dir) + release.tag_new_version() + except Exception as error: + if logger.getEffectiveLevel() == logging.DEBUG: + raise + raise Exit("An error occurred: {}".format(error)) from error From 47cb9750285aa2c794a015596d1ebc816964394b Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 18 Mar 2024 15:47:20 +0100 Subject: [PATCH 09/22] Fix docs. --- .github/workflows/build_job.yml | 5 +---- Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 4556288c..06cc1889 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -33,10 +33,6 @@ jobs: name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set docker version label - run: sed -i 's//${{ steps.repository.outputs.tag }}/g' Dockerfile - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -55,6 +51,7 @@ jobs: with: context: . platforms: linux/amd64 + build-args: VERSION=${{steps.repository.outputs.tag}} push: true tags: | ghcr.io/${{ steps.repository.outputs.repo }}:${{ steps.repository.outputs.tag }} diff --git a/Dockerfile b/Dockerfile index a06c1d39..a88e6d29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,11 @@ ARG CONDA_ENV_DIR=/opt/condaenv ARG FREVA_WEB_DIR=/opt/freva_web +ARG VERSION FROM condaforge/mambaforge LABEL org.opencontainers.image.authors="DRKZ-CLINT" LABEL org.opencontainers.image.source="https://github.com/FREVA-CLINT/freva-web" -LABEL org.opencontainers.image.version= +LABEL org.opencontainers.image.version=$VERSION ARG CONDA_ENV_DIR ARG FREVA_WEB_DIR From 6c159be0cd4513e1f00424a2944e30e5b3fcf917 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Mar 2024 04:14:59 +0100 Subject: [PATCH 10/22] Fix typo in Dockerfile. --- .github/workflows/build_job.yml | 2 -- Dockerfile | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 06cc1889..1455a700 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -6,8 +6,6 @@ run-name: ${{ github.actor }} is building the container push: tags: - '*' - branches: - - 'version' jobs: build-and-push-image: diff --git a/Dockerfile b/Dockerfile index a88e6d29..6bb1da4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG VERSION FROM condaforge/mambaforge LABEL org.opencontainers.image.authors="DRKZ-CLINT" LABEL org.opencontainers.image.source="https://github.com/FREVA-CLINT/freva-web" -LABEL org.opencontainers.image.version=$VERSION +LABEL org.opencontainers.image.version="$VERSION" ARG CONDA_ENV_DIR ARG FREVA_WEB_DIR From edabb66ea5a8b6620cf0889be0a735976f42d79e Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Mar 2024 04:30:45 +0100 Subject: [PATCH 11/22] Wait for docker-build job to finish lint. --- .github/workflows/build_job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 1455a700..bd6d87d5 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -10,6 +10,7 @@ run-name: ${{ github.actor }} is building the container jobs: build-and-push-image: runs-on: ubuntu-latest + needs: lint permissions: contents: read packages: write From 0d7b38945df6ac6afecbd51b6d2c88eb30beb432 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Mar 2024 05:17:50 +0100 Subject: [PATCH 12/22] Glue different workflows. --- .github/workflows/build_job.yml | 4 +++- .github/workflows/test_job.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index bd6d87d5..d644c9c5 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -5,9 +5,11 @@ run-name: ${{ github.actor }} is building the container workflow_dispatch: push: tags: - - '*' + - 'v*.*.*' jobs: + lint: + uses: .github/workflows/test_job.yml build-and-push-image: runs-on: ubuntu-latest needs: lint diff --git a/.github/workflows/test_job.yml b/.github/workflows/test_job.yml index 8b3ef7c9..7ab9fac2 100644 --- a/.github/workflows/test_job.yml +++ b/.github/workflows/test_job.yml @@ -1,7 +1,7 @@ name: Tests run-name: ${{ github.actor }} is doing some smoke tests -on: [push, pull_request] +on: [push, pull_request, workflow_call] jobs: lint: runs-on: ubuntu-latest From 6c9f940b8b4d49e92f72746a8b031dc9f1329623 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Mar 2024 05:23:02 +0100 Subject: [PATCH 13/22] Glue different workflows. --- .github/workflows/build_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index d644c9c5..7a75f3b7 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -9,7 +9,7 @@ run-name: ${{ github.actor }} is building the container jobs: lint: - uses: .github/workflows/test_job.yml + uses: ./.github/workflows/test_job.yml build-and-push-image: runs-on: ubuntu-latest needs: lint From 70cb11f33d7bb9b04c539d37658e5556dd610c42 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Wed, 20 Mar 2024 20:08:49 +0100 Subject: [PATCH 14/22] update: readme --- Makefile | 6 ++++-- README.md | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fcc52a9e..2bebc9d0 100644 --- a/Makefile +++ b/Makefile @@ -17,5 +17,7 @@ tests: pytest -vv $(PWD) tests/ release: - pip install git-python packaging tomli - python3 release.py tag django_evaluation -b version + pip install git-python requests packaging tomli + curl -H 'Cache-Control: no-cache' -Ls -o bump.py https://raw.githubusercontent.com/FREVA-CLINT/freva-deployment/versions/release.py + python3 bump.py tag django_evaluation -b version + rm bump.py diff --git a/README.md b/README.md index d8be35e9..27c2f0e5 100644 --- a/README.md +++ b/README.md @@ -119,12 +119,12 @@ it to the registry. To do so please follow the following steps. ```json "version": "2023.07.19" ``` -- After you have push the version changes to the main branch you can create - trigger the release procedure: +- After you have pushed the version changes to the main branch you can trigger + the release procedure: ```console make release ``` -This will check the current version of the `main` branch and created a trigger -a GitHub continuous integration pipeline to create a new release. The procedure +This will check the current version of the `main` branch and trigger +a GitHub continuous integration pipeline to create the new release. The procedure performs a couple of checks, if theses checks fail please make sure to address the issues. From fe0df6141bc5ee48a6ec446cd3431f0e7fa46034 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Wed, 20 Mar 2024 20:09:05 +0100 Subject: [PATCH 15/22] delete: release.py --- release.py | 224 ----------------------------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 release.py diff --git a/release.py b/release.py deleted file mode 100644 index 04a5cb8a..00000000 --- a/release.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Script that prepares a new release of a version.""" - -import argparse -import json -import logging -import re -import tempfile -from functools import cached_property -from itertools import product -from pathlib import Path - -import git -import tomli -from packaging.version import Version, InvalidVersion - -# Set up logging -logging.basicConfig( - format="%(name)s - %(levelname)s - %(message)s", - datefmt="[%X]", - level=logging.INFO, -) - - -logger = logging.getLogger("create-release") - - -class Exit(Exception): - """Custom error class for representing specific errors. - - Attributes: - message (str): A human-readable message describing the error. - code (int): An error code associated with the error. - """ - - def __init__(self, message: str, code: int = 1) -> None: - """Initialize the CustomError instance with the given message and error code. - - Args: - message (str): A human-readable message describing the error. - code (int): An error code associated with the error. - """ - super().__init__(message) - self.message = message - logger.critical(message) - raise SystemExit(code) - - -class Release: - """Release class.""" - - version_pattern: str = r'__version__\s*=\s*["\'](\d+\.\d+\.\d+)["\']' - - def __init__(self, package_name: str, repo_dir: str, branch: str = "main") -> None: - self.branch = branch - self.package_name = package_name - self.repo_dir = Path(repo_dir) - logger.info("Searching for packages/config with the name: %s", package_name) - logger.debug("Reading current git config") - self.git_config = ( - Path(git.Repo(search_parent_directories=True).git_dir) / "config" - ).read_text() - - def tag_version(self) -> None: - """Tag the latest git version.""" - - @cached_property - def repo_url(self) -> str: - """Get the current git repo url.""" - repo = git.Repo(search_parent_directories=True) - return repo.remotes.origin.url - - @cached_property - def git_tag(self) -> Version: - """Get the latest git tag.""" - logger.debug("Searching for the latest tag") - repo = git.Repo(self.repo_dir) - try: - # Get the latest tag on the main branch - return Version( - repo.git.describe("--tags", "--abbrev=0", self.branch).lstrip("v") - ) - except git.exc.GitCommandError: - logger.debug("No tag found") - except InvalidVersion: - logger.debug("Tag found, but could not parse version") - return Version("0.0.0") - - @property - def version(self) -> Version: - """Get the version of the current software.""" - logger.debug("Searching for software version.") - pck_dirs = Path("src") / self.package_name, Path( - "src" - ) / self.package_name.replace("-", "_") - files = [ - self.repo_dir / f[1] / f[0] - for f in product(("_version.py", "__init__.py"), pck_dirs) - ] - files += [ - self.repo_dir / Path("package.json"), - self.repo_dir / "pyproject.toml", - ] - for file in files: - if file.is_file(): - if file.suffix == ".py": - match = re.search(self.version_pattern, file.read_text()) - if match: - return Version(match.group(1)) - elif file.suffix == ".json": - content = json.loads(file.read_text()) - if "version" in content: - return Version(content["version"]) - elif file.suffix == ".toml": - content = tomli.loads(file.read_text()) - if "project" in content: - return Version(content["project"]["version"]) - raise ValueError("Could not find version") - - def _clone_repo_from_franch(self, branch: str = "main") -> None: - logger.debug( - "Cloning repository from %s with branch %s to %s", - self.repo_url, - self.repo_dir, - branch, - ) - git.Repo.clone_from(self.repo_url, self.repo_dir, branch=branch) - (self.repo_dir / ".git" / "config").write_text(self.git_config) - - def _check_change_lock_file(self) -> None: - """Check if the current version was added to the change lock file.""" - logger.debug("Checking for change log file.") - if not self._change_lock_file.is_file(): - raise Exit( - "Could not find change log file. " - f"Create one first and push it to the {self.branch} branch." - ) - if f"v{self.version}" not in self._change_lock_file.read_text("utf-8"): - raise Exit( - "You need to add the version v{} to the {} change log file " - "and push the update to the {} branch".format( - self.version, - self._change_lock_file.relative_to(self.repo_dir), - self.branch, - ) - ) - - @cached_property - def _change_lock_file(self) -> Path: - """Find the change lock file.""" - for prefix, suffix in product(("changelog", "whats-new"), (".rst", ".md")): - for search_pattern in (prefix, prefix.upper()): - glob_pattern = f"{search_pattern}{suffix}" - logger.debug("Searching for %s", glob_pattern) - for file in self.repo_dir.rglob(glob_pattern): - return file - return Path(tempfile.mktemp()) - - def tag_new_version(self) -> None: - """Tag a new git version.""" - self._clone_repo_from_franch(self.branch) - cloned_repo = git.Repo(self.repo_dir) - if self.version <= self.git_tag: - raise Exit( - "Tag version: {} is the same as current version {}" - ", you need to bump the verion number first and " - "push the changes to the {} branch".format( - self.version, - self.git_tag, - self.branch, - ) - ) - self._check_change_lock_file() - logger.info("Creating tag for version v%s", self.version) - head = cloned_repo.head.reference - message = f"Create a release for v{self.version}" - try: - cloned_repo.create_tag(f"v{self.version}", ref=head, message=message) - cloned_repo.git.push("--tags") - except git.GitCommandError as error: - raise Exit("Could not create tag: {}".format(error)) - logger.info("Tags created.") - - @classmethod - def cli(cls, temp_dir: str) -> "Release": - """Command line interface.""" - - parser = argparse.ArgumentParser( - description="Prepare the release of a package." - ) - subparser = parser.add_subparsers(help="Available commands:") - tag_parser = subparser.add_parser("tag", help="Create a new tag") - deploy_parser = subparser.add_parser( - "deploy", help="Update the version in the deployment repository" - ) - for _parser in tag_parser, deploy_parser: - _parser.add_argument("name", help="The name of the software/package.") - _parser.add_argument( - "-v", - "--verbose", - help="Enable debug mode.", - action="store_true", - ) - tag_parser.add_argument( - "-b", - "--branch", - help="Set the working branch", - type=str, - default="main", - ) - args = parser.parse_args() - if args.verbose: - logger.setLevel(logging.DEBUG) - return cls(args.name, temp_dir, args.branch) - - -if __name__ == "__main__": - with tempfile.TemporaryDirectory() as temporary_dir: - try: - release = Release.cli(temporary_dir) - release.tag_new_version() - except Exception as error: - if logger.getEffectiveLevel() == logging.DEBUG: - raise - raise Exit("An error occurred: {}".format(error)) from error From 5a0c9fd1456f7b8c4a20d3c877b1123f9857fce6 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Wed, 20 Mar 2024 20:15:37 +0100 Subject: [PATCH 16/22] update: build_job.yml --- .github/workflows/build_job.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index 7a75f3b7..30265baa 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -1,5 +1,5 @@ name: Build and publish the container image -run-name: ${{ github.actor }} is building the container +run-name: ${{ github.actor }} is createing a new release "on": workflow_dispatch: @@ -57,3 +57,32 @@ jobs: tags: | ghcr.io/${{ steps.repository.outputs.repo }}:${{ steps.repository.outputs.tag }} ghcr.io/${{ steps.repository.outputs.repo }}:latest + + bump-web-version: + runs-on: ubuntu-latest + needs: [build-and-push-image, lint] + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python 3 + uses: actions/setup-python@v4 + with: + python-version: "3.X" + + - name: Get tag + id: repository + run: | + echo "repo=$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')" >> $GITHUB_OUTPUT + echo "tag=$(cat package.json|jq -r .version)" >> $GITHUB_OUTPUT + + - name: Do the release job + run: make release + env: + GITHUB_TOKEN: ${{secrets.ACCESS_TOKEN}} + REPO_VERSION: ${{ steps.repository.outputs.tag }} From 77f58594db59fa3bda8697905d862a85ba789929 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Sat, 23 Mar 2024 18:58:16 +0100 Subject: [PATCH 17/22] Set GID to 1000 as id's beyond 65k don't work in user mode --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6bb1da4f..e7629f5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,8 @@ ARG CONDA_ENV_DIR ARG FREVA_WEB_DIR RUN set -e && \ - groupadd -r -g 200855 freva && \ - adduser --uid 1000 --gid 200855 --gecos "Freva user" \ + groupadd -r -g 1000 freva && \ + adduser --uid 1000 --gid 1000 --gecos "Freva user" \ --shell /bin/bash --disabled-password freva --home ${FREVA_WEB_DIR} &&\ mkdir -p ${CONDA_ENV_DIR} && chown -R freva:freva $CONDA_ENV_DIR WORKDIR ${FREVA_WEB_DIR} From fde91c91ecc117e574ff473db7e3057ba378e33a Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 25 Mar 2024 07:16:02 +0100 Subject: [PATCH 18/22] Use autouse flag for eval config. --- tests/conftest.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 98f21e4b..e93708f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,14 @@ -from configparser import ConfigParser, ExtendedInterpolation import os -from pathlib import Path -import pytest -import mock import random import string import sys +from configparser import ConfigParser, ExtendedInterpolation +from pathlib import Path from tempfile import NamedTemporaryFile +import mock +import pytest + def get_config(): from evaluation_system.tests.mocks import TEST_EVAL @@ -36,7 +37,7 @@ def get_config(): return test_cfg -@pytest.fixture(scope="session") +@pytest.fixture(scope="session", autouse=True) def eval_pubkey(): with NamedTemporaryFile(suffix=".crt") as tf: with Path(tf.name).open("w") as f: @@ -44,7 +45,7 @@ def eval_pubkey(): yield tf.name -@pytest.fixture(scope="session") +@pytest.fixture(scope="session", autouse=True) def eval_config(eval_pubkey): config = get_config() path_prefix = Path(sys.exec_prefix) / "bin" From f43ec5d1c52c5af82bb5228482996240bd75b36d Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 25 Mar 2024 07:16:21 +0100 Subject: [PATCH 19/22] Use tests. --- tests/test_manage.py | 53 +++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/tests/test_manage.py b/tests/test_manage.py index adcb7978..962ede31 100644 --- a/tests/test_manage.py +++ b/tests/test_manage.py @@ -1,26 +1,49 @@ import shlex -from subprocess import run, PIPE +from subprocess import PIPE, run -def test_chek(eval_config): +def run_cmd(inp: str) -> int: + """Run a python manage.py command.""" + cmd = f"python manage.py {inp}" + res = run( + shlex.split(cmd), + check=False, + stdout=PIPE, + stderr=PIPE, + encoding="utf-8", + ) + if res.returncode != 0: + print("================== STDOUT ============================") + print(res.stdout) + print("================== STDERR ============================") + print(res.stderr) + return res.returncode + + +def test_migrations(): + """Test migrations.""" + assert run_cmd("makemigrations base") == 0 + assert run_cmd("migrate --fake-initial --noinput") == 0 + assert run_cmd("migrate --fake contenttypes") == 0 + assert run_cmd("collectstatic --noinput") == 0 + + +def test_chek(): """Test the python manage.py check cmd""" - cmd = f"python manage.py check" - res = run(shlex.split(cmd), check=False, stdout=PIPE, stderr=PIPE) - assert res.returncode == 0 + assert run_cmd("check") == 0 -def test_collectstatic(eval_config): +def test_collectstatic(): """Test the python manage.py collectstatic command.""" - cmd = f"python manage.py collectstatic -n --noinput" - res = run(shlex.split(cmd), check=False, stdout=PIPE, stderr=PIPE) - assert res.returncode == 0 + assert run_cmd("collectstatic -n --noinput") == 0 -def test_createsuperuser(eval_config, random_admin): +def test_createsuperuser(random_admin): """The the python manage.py createsuperuser command.""" - cmd = ( - f"python manage.py createsuperuser --username {random_admin} " - "--email test@bla.com --noinput" + assert ( + run_cmd( + f"createsuperuser --username {random_admin} " + "--email test@bla.com --noinput" + ) + == 0 ) - res = run(shlex.split(cmd), check=False, stdout=PIPE, stderr=PIPE) - assert res.returncode == 0 From 06e8ba7d4ca3ed64c7bffe9616857d536a87567b Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 25 Mar 2024 07:16:35 +0100 Subject: [PATCH 20/22] Install isort. --- conda-env.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-env.yml b/conda-env.yml index 930c32ab..defd22a9 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -30,6 +30,7 @@ dependencies: - pip - pip: - black + = isort - requests_mock - django-auth-ldap - django-datatable-view From 979cffca494bab8d0a58775859780495a527f465 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 25 Mar 2024 07:19:19 +0100 Subject: [PATCH 21/22] Install isort. --- conda-env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-env.yml b/conda-env.yml index defd22a9..4d4e8445 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -30,7 +30,7 @@ dependencies: - pip - pip: - black - = isort + - isort - requests_mock - django-auth-ldap - django-datatable-view From 9f7d21ba97579778dc34ddb095c400f58c5d9c0a Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 25 Mar 2024 07:24:57 +0100 Subject: [PATCH 22/22] Linting with isort and black. --- .ci/install_conda.py | 5 ++-- Makefile | 2 +- base/LdapUser.py | 7 ++++-- base/admin.py | 4 +-- base/models.py | 6 ++--- base/serializers.py | 2 +- base/urls.py | 1 + base/views.py | 17 +++++++------ base/views_api.py | 5 ++-- django_evaluation/ldaptools.py | 14 ++++++----- django_evaluation/monitor.py | 6 ++--- django_evaluation/settings/base.py | 5 ++-- django_evaluation/settings/local.py | 14 +++++------ django_evaluation/settings/test.py | 1 - django_evaluation/urls.py | 10 +++++--- django_evaluation/utils.py | 1 + django_evaluation/wsgi.py | 3 ++- docker/solr/ingest_dummy_data.py | 3 +-- dummy_plugin/dummy.py | 15 ++++++----- history/admin.py | 3 ++- history/models.py | 2 +- history/templatetags/dialogtags.py | 7 +++--- history/templatetags/resulttags.py | 17 ++++++------- history/urls.py | 4 +-- history/utils.py | 5 ++-- history/views.py | 39 ++++++++++++----------------- history/views_api.py | 10 ++++---- manage.py | 3 ++- plugins/forms.py | 12 +++++---- plugins/models.py | 2 +- plugins/serializers.py | 2 +- plugins/urls.py | 4 ++- plugins/utils.py | 11 ++++---- plugins/views.py | 36 +++++++++++--------------- plugins/views_api.py | 14 ++++++----- solr/admin.py | 1 + solr/templatetags/custom_filters.py | 5 ++-- solr/urls.py | 7 +++--- solr/views.py | 11 ++++---- solr/views_api.py | 8 +++--- tests/website_unittest.py | 5 ++-- 41 files changed, 166 insertions(+), 163 deletions(-) diff --git a/.ci/install_conda.py b/.ci/install_conda.py index 6f26c814..2e33897a 100755 --- a/.ci/install_conda.py +++ b/.ci/install_conda.py @@ -2,13 +2,12 @@ import argparse import logging -from pathlib import Path import shlex -from subprocess import CalledProcessError, PIPE, run import urllib.request +from pathlib import Path +from subprocess import PIPE, CalledProcessError, run from tempfile import TemporaryDirectory - MINICONDA_URL = "https://repo.anaconda.com/miniconda/Miniconda3-latest" logging.basicConfig(format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO) diff --git a/Makefile b/Makefile index 5a5d0691..c5c1c1f5 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ run: runfrontend runserver lint: setup-node npm run lint-format npm run lint - black -t py310 --check . + isort -c --profile black -t py312 . tests: setup-node npm run build-production diff --git a/base/LdapUser.py b/base/LdapUser.py index 726d66c9..411ae49d 100644 --- a/base/LdapUser.py +++ b/base/LdapUser.py @@ -1,10 +1,13 @@ import os +from configparser import ConfigParser as Config +from configparser import ExtendedInterpolation + +from evaluation_system.misc import config from evaluation_system.model.db import UserDB from evaluation_system.model.user import User + from base.exceptions import UserNotFoundError from django_evaluation.ldaptools import get_ldap_object -from configparser import ConfigParser as Config, ExtendedInterpolation -from evaluation_system.misc import config class LdapUser(User): diff --git a/base/admin.py b/base/admin.py index 374103e7..01d8e7a1 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,8 +1,6 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin -from django.contrib.auth.models import User - -from django.contrib.auth.models import Permission +from django.contrib.auth.models import Permission, User from .models import UIMessages diff --git a/base/models.py b/base/models.py index 94d8c85a..372d2ca9 100644 --- a/base/models.py +++ b/base/models.py @@ -1,8 +1,8 @@ -from django.db import models -from django.contrib.auth.models import User - from datetime import datetime +from django.contrib.auth.models import User +from django.db import models + def isGuest(self): groups = self.groups.filter(name="Guest") diff --git a/base/serializers.py b/base/serializers.py index 3784a430..13de182b 100644 --- a/base/serializers.py +++ b/base/serializers.py @@ -1,5 +1,5 @@ -from rest_framework import serializers from django.contrib.auth.models import User +from rest_framework import serializers from django_evaluation.settings.local import HOME_DIRS_AVAILABLE diff --git a/base/urls.py b/base/urls.py index 0a46b3a4..2338fc83 100644 --- a/base/urls.py +++ b/base/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + import base.views urlpatterns = [ diff --git a/base/views.py b/base/views.py index 8a185315..c4d8f6b3 100644 --- a/base/views.py +++ b/base/views.py @@ -1,17 +1,17 @@ import logging -from base.models import UIMessages - -from django.http import HttpResponseRedirect -from django.shortcuts import render import django.contrib.auth as auth -from django.contrib.auth.decorators import login_required, user_passes_test -from django.views.decorators.debug import sensitive_variables, sensitive_post_parameters -from django_evaluation.monitor import _restart from django.conf import settings +from django.contrib.auth.decorators import login_required, user_passes_test +from django.http import HttpResponseRedirect +from django.shortcuts import render from django.urls import reverse -from evaluation_system.misc import config from django.utils.http import url_has_allowed_host_and_scheme +from django.views.decorators.debug import sensitive_post_parameters, sensitive_variables +from evaluation_system.misc import config + +from base.models import UIMessages +from django_evaluation.monitor import _restart @sensitive_variables("passwd") @@ -113,6 +113,7 @@ def contact(request): """ if request.method == "POST": from templated_email import send_templated_mail + from django_evaluation.ldaptools import get_ldap_object user_info = get_ldap_object() diff --git a/base/views_api.py b/base/views_api.py index 089e0277..b935a91f 100644 --- a/base/views_api.py +++ b/base/views_api.py @@ -1,8 +1,9 @@ -from rest_framework.views import APIView from rest_framework.response import Response +from rest_framework.views import APIView -from base.LdapUser import LdapUser from base.exceptions import UserNotFoundError +from base.LdapUser import LdapUser + from .serializers import UserSerializer diff --git a/django_evaluation/ldaptools.py b/django_evaluation/ldaptools.py index 3650a143..33f31b34 100644 --- a/django_evaluation/ldaptools.py +++ b/django_evaluation/ldaptools.py @@ -1,16 +1,18 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod + +import grp import importlib +import pwd import time +from abc import ABCMeta, abstractmethod from typing import Iterator -from django_evaluation import settings -from django_evaluation.utils import background +import ldap from django.core.cache import cache, caches from django.core.exceptions import ImproperlyConfigured -import ldap -import grp -import pwd + +from django_evaluation import settings +from django_evaluation.utils import background """ FIXME: 28.03.2022 diff --git a/django_evaluation/monitor.py b/django_evaluation/monitor.py index 5f3a052f..c599e55f 100644 --- a/django_evaluation/monitor.py +++ b/django_evaluation/monitor.py @@ -3,12 +3,12 @@ http://code.google.com/p/modwsgi/wiki/ReloadingSourceCode#Restarting_Daemon_Processes """ +import atexit import os -import sys +import queue import signal +import sys import threading -import atexit -import queue _interval = 1.0 _times = {} diff --git a/django_evaluation/settings/base.py b/django_evaluation/settings/base.py index e320ce83..8713a8dd 100755 --- a/django_evaluation/settings/base.py +++ b/django_evaluation/settings/base.py @@ -3,15 +3,16 @@ repo. If you need to override a setting locally, use local.py """ -import os import logging +import os from pathlib import Path +import django.utils + # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. from django.core.exceptions import ImproperlyConfigured from django.urls import reverse_lazy -import django.utils def get_env_setting(setting): diff --git a/django_evaluation/settings/local.py b/django_evaluation/settings/local.py index 37e6bb33..df3bdaa4 100755 --- a/django_evaluation/settings/local.py +++ b/django_evaluation/settings/local.py @@ -1,17 +1,15 @@ -from pathlib import Path import os +import shutil import sys +from pathlib import Path + import ldap -from django_auth_ldap.config import ( - LDAPSearch, - NestedGroupOfNamesType, - PosixGroupType, -) -import shutil -import toml import requests +import toml from django.urls import reverse_lazy +from django_auth_ldap.config import LDAPSearch, NestedGroupOfNamesType, PosixGroupType from evaluation_system.misc import config + from base.exceptions import UnknownLDAPGroupTypeError freva_share_path = Path(os.environ["EVALUATION_SYSTEM_CONFIG_FILE"]).parent diff --git a/django_evaluation/settings/test.py b/django_evaluation/settings/test.py index 3c127387..11512d26 100644 --- a/django_evaluation/settings/test.py +++ b/django_evaluation/settings/test.py @@ -6,7 +6,6 @@ from .base import * - TEST_RUNNER = "discover_runner.DiscoverRunner" TEST_DISCOVER_TOP_LEVEL = PROJECT_ROOT TEST_DISCOVER_ROOT = PROJECT_ROOT diff --git a/django_evaluation/urls.py b/django_evaluation/urls.py index 70d767c0..74b53764 100644 --- a/django_evaluation/urls.py +++ b/django_evaluation/urls.py @@ -1,19 +1,21 @@ """ Default urlconf for django_evaluation """ from django.conf import settings -from django.urls import include, re_path as url from django.conf.urls import static -from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.contrib import admin +from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from django.urls import include +from django.urls import re_path as url from django.views.generic import RedirectView + from base.views_api import AuthenticatedUser +from history.views_api import ResultFacets, ResultFiles from plugins.views_api import ( - PluginsList, ExportPlugin, PluginDetail, + PluginsList, SendMailToDeveloper, ) -from history.views_api import ResultFacets, ResultFiles from solr.views_api import ncdump admin.autodiscover() diff --git a/django_evaluation/utils.py b/django_evaluation/utils.py index 40c3cb7d..ebc77bbc 100644 --- a/django_evaluation/utils.py +++ b/django_evaluation/utils.py @@ -1,6 +1,7 @@ """Collection of utility functions.""" from __future__ import annotations + import threading from typing import Any, Callable diff --git a/django_evaluation/wsgi.py b/django_evaluation/wsgi.py index 268786cc..6b475cf8 100644 --- a/django_evaluation/wsgi.py +++ b/django_evaluation/wsgi.py @@ -15,8 +15,8 @@ """ import os -import sys import site +import sys PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__) + "../../") @@ -48,6 +48,7 @@ # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. from django.core.wsgi import get_wsgi_application + from django_evaluation.ldaptools import get_ldap_object LdapCls = get_ldap_object() diff --git a/docker/solr/ingest_dummy_data.py b/docker/solr/ingest_dummy_data.py index d9feee30..992916fa 100644 --- a/docker/solr/ingest_dummy_data.py +++ b/docker/solr/ingest_dummy_data.py @@ -3,9 +3,8 @@ import logging from pathlib import Path -from evaluation_system.model.solr_core import SolrCore from evaluation_system.misc import config, logger - +from evaluation_system.model.solr_core import SolrCore if __name__ == "__main__": logger.setLevel(logging.INFO) diff --git a/dummy_plugin/dummy.py b/dummy_plugin/dummy.py index dff086c7..885977c0 100644 --- a/dummy_plugin/dummy.py +++ b/dummy_plugin/dummy.py @@ -1,18 +1,17 @@ -import tempfile -import shutil import os +import shutil +import tempfile -from evaluation_system.api.plugin import PluginAbstract from evaluation_system.api.parameters import ( - ParameterDictionary, - Integer, + Directory, Float, + Integer, + ParameterDictionary, String, - Directory, ) - -from evaluation_system.model.user import User +from evaluation_system.api.plugin import PluginAbstract from evaluation_system.model.db import UserDB +from evaluation_system.model.user import User class DummyPlugin(PluginAbstract): diff --git a/history/admin.py b/history/admin.py index 18605b96..54ed5478 100644 --- a/history/admin.py +++ b/history/admin.py @@ -1,8 +1,9 @@ from django.contrib import admin -from history.models import History, Configuration from django.contrib.auth.models import User from django.utils.html import format_html +from history.models import Configuration, History + class HistoryAmdin(admin.ModelAdmin): list_display = ( diff --git a/history/models.py b/history/models.py index 8df194b4..24001803 100644 --- a/history/models.py +++ b/history/models.py @@ -1,7 +1,7 @@ from evaluation_system.model.history.models import ( + Configuration, History, HistoryTag, Result, ResultTag, - Configuration, ) diff --git a/history/templatetags/dialogtags.py b/history/templatetags/dialogtags.py index 556a101c..5016533c 100644 --- a/history/templatetags/dialogtags.py +++ b/history/templatetags/dialogtags.py @@ -1,9 +1,10 @@ +import json + from django import template -from django_evaluation.ldaptools import get_ldap_object from django.utils.safestring import mark_safe -from django_evaluation import settings -import json +from django_evaluation import settings +from django_evaluation.ldaptools import get_ldap_object register = template.Library() diff --git a/history/templatetags/resulttags.py b/history/templatetags/resulttags.py index 66c0cc0e..e65103f3 100644 --- a/history/templatetags/resulttags.py +++ b/history/templatetags/resulttags.py @@ -1,18 +1,17 @@ +import re + from django import template -from django.utils.safestring import mark_safe +from django.template.loader import render_to_string from django.utils.encoding import force_str from django.utils.html import conditional_escape -from django.template.loader import render_to_string -from base.LdapUser import LdapUser +from django.utils.safestring import mark_safe +from evaluation_system.misc.exceptions import PluginManagerException + from base.exceptions import UserNotFoundError +from base.LdapUser import LdapUser from django_evaluation import settings - -from history.utils import FileDict from history.models import HistoryTag - -from evaluation_system.misc.exceptions import PluginManagerException - -import re +from history.utils import FileDict register = template.Library() diff --git a/history/urls.py b/history/urls.py index 80844ee8..8be2049a 100644 --- a/history/urls.py +++ b/history/urls.py @@ -1,10 +1,10 @@ -from django.urls import re_path as url from django.contrib.auth.decorators import login_required +from django.urls import re_path as url from django.views.decorators.cache import never_cache + from history import views from plugins.views_api import ShareResultsByMail - urlpatterns = [ url( r"^$", never_cache(login_required(views.HistoryTable.as_view())), name="history" diff --git a/history/utils.py b/history/utils.py index 645d331d..986168ce 100644 --- a/history/utils.py +++ b/history/utils.py @@ -1,9 +1,9 @@ import os -from evaluation_system.misc import utils +from evaluation_system.misc import config, utils from evaluation_system.model.history.models import History, HistoryTag + from django_evaluation import settings -from evaluation_system.misc import config from django_evaluation.settings.local import SERVER_EMAIL @@ -151,6 +151,7 @@ def sendmail_to_follower(request, history_id, subject, message): :param message: the mesaage """ from django.urls import reverse + from django_evaluation.ldaptools import get_ldap_object follower = HistoryTag.objects.filter(history_id_id=history_id).filter( diff --git a/history/views.py b/history/views.py index 10ae02ea..d5717373 100644 --- a/history/views.py +++ b/history/views.py @@ -3,39 +3,32 @@ import os from pathlib import Path -from django.shortcuts import render -from django.http import HttpResponse -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist +import evaluation_system.api.plugin_manager as pm +from datatableview import Datatable, columns +from datatableview.views import DatatableView from django.contrib.auth.decorators import login_required -from django.views.decorators.debug import sensitive_post_parameters from django.contrib.flatpages.models import FlatPage +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied +from django.db.models import Q +from django.http import HttpResponse +from django.shortcuts import get_object_or_404, render +from django.urls import reverse from django.utils.html import escape -from datatableview import columns -from datatableview import Datatable -from datatableview.views import DatatableView -from django.utils.html import escape - - -import evaluation_system.api.plugin_manager as pm -from evaluation_system.model.db import UserDB +from django.views.decorators.debug import sensitive_post_parameters from evaluation_system.api.workload_manager import get_job_class -from evaluation_system.model.user import User -from evaluation_system.misc.exceptions import PluginManagerException from evaluation_system.misc import config as eval_config +from evaluation_system.misc.exceptions import PluginManagerException +from evaluation_system.model.db import UserDB from evaluation_system.model.history.models import History, ResultTag -from base.LdapUser import LdapUser +from evaluation_system.model.user import User + from base.exceptions import UserNotFoundError +from base.LdapUser import LdapUser from django_evaluation import settings -from plugins.utils import ssh_call, get_scheduler_hosts - -from history.utils import FileDict, sendmail_to_follower - -from django.shortcuts import get_object_or_404 -from django.db.models import Q -from django.urls import reverse - from history.models import HistoryTag from history.templatetags.resulttags import mask_uid +from history.utils import FileDict, sendmail_to_follower +from plugins.utils import get_scheduler_hosts, ssh_call class HistoryDatatable(Datatable): diff --git a/history/views_api.py b/history/views_api.py index 1f6bb356..bb8ba7d8 100644 --- a/history/views_api.py +++ b/history/views_api.py @@ -1,14 +1,14 @@ import json import re + +from django.conf import settings +from django.core.cache import cache from django.db.models import Q +from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page -from django.conf import settings -from django.urls import reverse -from django.core.cache import cache - -from rest_framework.views import APIView from rest_framework.response import Response +from rest_framework.views import APIView from history.models import History diff --git a/manage.py b/manage.py index bac9edc4..d5b4a8c3 100644 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -import os, sys +import os +import sys if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_evaluation.settings") diff --git a/plugins/forms.py b/plugins/forms.py index 7ef3f426..453edcda 100644 --- a/plugins/forms.py +++ b/plugins/forms.py @@ -1,14 +1,16 @@ -from django import forms +from pathlib import Path + import django.contrib.auth as auth +import evaluation_system.api.parameters as parameters +from django import forms +from django.conf import settings from django.core import exceptions from django.forms.widgets import Input, TextInput from django.template import loader -import evaluation_system.api.parameters as parameters from evaluation_system.api import plugin_manager as pm -from pathlib import Path -from plugins.utils import ssh_call from evaluation_system.misc.utils import PrintableList -from django.conf import settings + +from plugins.utils import ssh_call class PluginNotFoundError(Exception): diff --git a/plugins/models.py b/plugins/models.py index 8e093048..f8495dcf 100644 --- a/plugins/models.py +++ b/plugins/models.py @@ -1 +1 @@ -from evaluation_system.model.plugins.models import Version, Parameter +from evaluation_system.model.plugins.models import Parameter, Version diff --git a/plugins/serializers.py b/plugins/serializers.py index 26f9106a..41ce5a9e 100644 --- a/plugins/serializers.py +++ b/plugins/serializers.py @@ -1,5 +1,5 @@ -from rest_framework import serializers from django.contrib.flatpages.models import FlatPage +from rest_framework import serializers class PluginSerializer(serializers.Serializer): diff --git a/plugins/urls.py b/plugins/urls.py index 2422d451..d78ea9a8 100644 --- a/plugins/urls.py +++ b/plugins/urls.py @@ -1,4 +1,6 @@ -from django.urls import re_path as url, include +from django.urls import include +from django.urls import re_path as url + import plugins.views urlpatterns = [ diff --git a/plugins/utils.py b/plugins/utils.py index 6dfbd4ce..f526c096 100644 --- a/plugins/utils.py +++ b/plugins/utils.py @@ -1,11 +1,12 @@ import base64 +import logging + import evaluation_system.api.plugin_manager as pm -from evaluation_system.misc import config -from django.views.decorators.debug import sensitive_variables -from django.http import Http404 -from django.conf import settings import paramiko -import logging +from django.conf import settings +from django.http import Http404 +from django.views.decorators.debug import sensitive_variables +from evaluation_system.misc import config def get_scheduler_hosts(user): diff --git a/plugins/views.py b/plugins/views.py index 815b761f..c73493c0 100644 --- a/plugins/views.py +++ b/plugins/views.py @@ -1,37 +1,31 @@ """ Views for the plugins application """ -from django.shortcuts import render, redirect -from django.http import HttpResponse -from django.http import JsonResponse -from django.contrib.auth.decorators import login_required -from django.conf import settings -from django.views.decorators.debug import ( - sensitive_variables, - sensitive_post_parameters, -) +import json +import logging +import os +import urllib from pathlib import Path import evaluation_system.api.plugin_manager as pm - -from evaluation_system.model.user import User +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.http import HttpResponse, JsonResponse +from django.shortcuts import redirect, render +from django.views.decorators.debug import sensitive_post_parameters, sensitive_variables from evaluation_system.misc import config -from django_evaluation.settings.local import HOME_DIRS_AVAILABLE +from evaluation_system.model.user import User + from base.exceptions import UserNotFoundError from base.LdapUser import LdapUser - +from django_evaluation.settings.local import HOME_DIRS_AVAILABLE +from history.models import Configuration, History +from plugins.forms import PluginForm, PluginWeb from plugins.utils import ( get_plugin_or_404, + get_scheduler_hosts, is_path_relative_to, ssh_call, - get_scheduler_hosts, ) -from plugins.forms import PluginForm, PluginWeb -from history.models import History, Configuration - -import logging -import urllib -import os -import json @login_required() diff --git a/plugins/views_api.py b/plugins/views_api.py index f8087e3f..2b114d50 100644 --- a/plugins/views_api.py +++ b/plugins/views_api.py @@ -1,17 +1,19 @@ -from rest_framework.views import APIView -from rest_framework.response import Response -from rest_framework.permissions import IsAuthenticated import os import evaluation_system.api.plugin_manager as pm -from evaluation_system.model.user import User +from django.conf import settings from evaluation_system.misc import config -from django_evaluation.ldaptools import get_ldap_object +from evaluation_system.model.user import User +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.views import APIView + from base.exceptions import UserNotFoundError from base.LdapUser import LdapUser +from django_evaluation.ldaptools import get_ldap_object from plugins.utils import get_plugin_or_404, plugin_metadata_as_dict + from .serializers import PluginSerializer -from django.conf import settings class PluginsList(APIView): diff --git a/solr/admin.py b/solr/admin.py index b9408e17..995796f1 100644 --- a/solr/admin.py +++ b/solr/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from solr.models import UserCrawl diff --git a/solr/templatetags/custom_filters.py b/solr/templatetags/custom_filters.py index dc05da2e..2c736783 100644 --- a/solr/templatetags/custom_filters.py +++ b/solr/templatetags/custom_filters.py @@ -1,9 +1,8 @@ -from django import template +import urllib.parse +from django import template from django.utils.safestring import mark_safe -import urllib.parse - register = template.Library() diff --git a/solr/urls.py b/solr/urls.py index 4c735441..cc3769d7 100644 --- a/solr/urls.py +++ b/solr/urls.py @@ -7,11 +7,12 @@ """ import os -from django.urls import re_path as url -from django.urls import path from django.conf import settings -from .views import databrowser +from django.urls import path +from django.urls import re_path as url + from .proxyviews import DataBrowserProxy +from .views import databrowser urlpatterns = [ url(r"^databrowser/$", databrowser, name="data_browser"), diff --git a/solr/views.py b/solr/views.py index 528e721d..dfd93104 100644 --- a/solr/views.py +++ b/solr/views.py @@ -6,15 +6,14 @@ views for the solr application """ +import logging from typing import Union -import logging -from django.shortcuts import render -from django.contrib.auth.decorators import login_required -from django.http import JsonResponse -from django.conf import settings -from django.http import QueryDict import requests +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.http import JsonResponse, QueryDict +from django.shortcuts import render @login_required() diff --git a/solr/views_api.py b/solr/views_api.py index e8649383..57a56a8f 100644 --- a/solr/views_api.py +++ b/solr/views_api.py @@ -1,13 +1,13 @@ import logging -from rest_framework.decorators import api_view +from django.conf import settings from django.contrib.auth.decorators import login_required -from django.utils.safestring import mark_safe from django.http import JsonResponse -from django.conf import settings +from django.utils.safestring import mark_safe from paramiko import AuthenticationException +from rest_framework.decorators import api_view -from plugins.utils import ssh_call, get_scheduler_hosts +from plugins.utils import get_scheduler_hosts, ssh_call @api_view(["POST"]) diff --git a/tests/website_unittest.py b/tests/website_unittest.py index c5b94a76..d0837c66 100644 --- a/tests/website_unittest.py +++ b/tests/website_unittest.py @@ -1,8 +1,9 @@ -import unittest import time +import unittest + from selenium import webdriver -from selenium.webdriver.common.keys import Keys from selenium.common.exceptions import NoSuchElementException +from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.select import Select