From 1dadcfa4897b5f9897aaaa1969ac60919b873b07 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 28 Feb 2022 13:20:59 +0100 Subject: [PATCH 01/34] + added clamav to docker-compose --- Makefile | 2 ++ docker-compose.override.dev.yml | 4 ++++ docker-compose.yml | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/Makefile b/Makefile index bf65b513..92e3ffe2 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ start-dev: docker-compose up -d \ portfolio-redis \ portfolio-postgres \ + portfolio-clamav \ portfolio-lool start-dev-docker: @@ -55,6 +56,7 @@ start-dev-docker: docker-compose up -d \ portfolio-redis \ portfolio-postgres \ + portfolio-clamav \ portfolio-lool \ portfolio-django docker logs -f portfolio-django diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index c17874c7..2dd58eb8 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -12,6 +12,10 @@ services: ports: - "127.0.0.1:8200:8200" + portfolio-clamav: + ports: + - "127.0.0.1:3310:3310" + portfolio-lool: ports: - "127.0.0.1:9980:9980" diff --git a/docker-compose.yml b/docker-compose.yml index 8ba18be9..446d246f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,6 +99,13 @@ services: restart: always command: python manage.py rqscheduler + portfolio-clamav: + image: clamav/clamav:stable + container_name: portfolio-clamav + networks: + - portfolionet + restart: always + portfolio-lool: image: collabora/code:4.2.3.1 container_name: portfolio-lool From a1757b5db479648cdd465d82722a4c218dec4066 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 28 Feb 2022 13:22:28 +0100 Subject: [PATCH 02/34] + added clamd to requirements --- src/requirements-dev.txt | 2 ++ src/requirements.in | 1 + src/requirements.txt | 2 ++ 3 files changed, 5 insertions(+) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index 6fd64f02..8fe7f117 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -20,6 +20,8 @@ cfgv==3.2.0 # via pre-commit chardet==4.0.0 # via requests +clamd==1.0.2 + # via -r src/requirements.in click==7.1.2 # via # pip-tools diff --git a/src/requirements.in b/src/requirements.in index 62db489b..79129d0f 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -6,6 +6,7 @@ apimapper==0.7.8 apispec==0.39.0 bibtexparser==1.2.0 concurrent-log-handler==0.9.19 +clamd~=1.0 django~=2.2 django-cas-ng==4.1.1 django-cleanup==5.2.0 diff --git a/src/requirements.txt b/src/requirements.txt index 263ea968..565f6a65 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -16,6 +16,8 @@ certifi==2020.12.5 # via requests chardet==4.0.0 # via requests +clamd==1.0.2 + # via -r src/requirements.in click==7.1.2 # via # pip-tools From b4ec130b19a589b26af24634e14c15b4dc46f5c5 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 28 Feb 2022 13:25:41 +0100 Subject: [PATCH 03/34] + added clamav validator + added clamav settings --- src/media_server/clamav.py | 39 ++++++++++++++++++++++++++++++++++++++ src/portfolio/settings.py | 4 ++++ 2 files changed, 43 insertions(+) create mode 100644 src/media_server/clamav.py diff --git a/src/media_server/clamav.py b/src/media_server/clamav.py new file mode 100644 index 00000000..d57ae5c5 --- /dev/null +++ b/src/media_server/clamav.py @@ -0,0 +1,39 @@ +# Parts of code from https://github.com/chander/django-clamav/ +import logging + +import clamd + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +logger = logging.getLogger(__name__) + + +def get_scanner(): + return clamd.ClamdNetworkSocket(settings.CLAMAV_TCP_ADDR, settings.CLAMAV_TCP_PORT) + + +def validate_file_infection(file): + if not settings.CLAMAV_ENABLED: + return + + # Ensure file pointer is at beginning of the file + file.seek(0) + + scanner = get_scanner() + try: + result = scanner.instream(file) + except OSError: + # Ping the server if it fails than the server is down + scanner.ping() + # Server is up. This means that the file is too big. + logger.warning(f'The file is too large for ClamD to scan it. Bytes Read {file.tell()}') + file.seek(0) + return + + if result and result['stream'][0] == 'FOUND': + raise ValidationError(_('File is infected with malware'), code='infected') + + # Return file pointer to beginning of the file again + file.seek(0) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index fb719a83..fd9dfa07 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -720,6 +720,10 @@ USER_QUOTA = env.int('USER_QUOTA', default=10 * 1024 * 1024 * 1024) # user quota / year +CLAMAV_ENABLED = env.bool('CLAMAV_ENABLED', default=True) +CLAMAV_TCP_PORT = env.int('CLAMAV_TCP_PORT', default=3310) +CLAMAV_TCP_ADDR = f'{PROJECT_NAME}-clamav' if DOCKER else 'localhost' + LOOL_HOST = 'http://{}:9980'.format(f'{PROJECT_NAME}-lool' if DOCKER else 'localhost') DOCS_USER = env('DOCS_USER', default=None) From 89aeb762c52781a2c9a71c3246fcd381615cb1cb Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 28 Feb 2022 13:27:09 +0100 Subject: [PATCH 04/34] + using clamav validator in model and serializer --- src/media_server/models.py | 5 ++++- src/media_server/serializers.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/media_server/models.py b/src/media_server/models.py index 2a0a83da..f31f70a3 100644 --- a/src/media_server/models.py +++ b/src/media_server/models.py @@ -20,6 +20,7 @@ from general.models import ShortUUIDField from .apps import MediaServerConfig +from .clamav import validate_file_infection from .storages import ProtectedFileSystemStorage from .utils import humanize_size, user_hash from .validators import validate_license @@ -114,7 +115,9 @@ def user_directory_path(instance, filename): class Media(models.Model): id = ShortUUIDField(primary_key=True) - file = models.FileField(storage=ProtectedFileSystemStorage(), upload_to=user_directory_path) + file = models.FileField( + storage=ProtectedFileSystemStorage(), upload_to=user_directory_path, validators=[validate_file_infection] + ) type = models.CharField(choices=TYPE_CHOICES, max_length=1, default=OTHER_TYPE) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) diff --git a/src/media_server/serializers.py b/src/media_server/serializers.py index 17ace8b5..1be0aea7 100644 --- a/src/media_server/serializers.py +++ b/src/media_server/serializers.py @@ -5,6 +5,7 @@ from core.models import Entry +from .clamav import validate_file_infection from .validators import validate_license as vl @@ -23,6 +24,13 @@ class MediaCreateSerializer(serializers.Serializer): published = serializers.BooleanField() license = serializers.JSONField() + def validate_file(self, value): + try: + validate_file_infection(value) + except ValidationError as e: + raise serializers.ValidationError(e.message) from e + return value + def validate_entry(self, value): try: entry = Entry.objects.get(id=value) From 34c0edd8c925e408e7576d06a46e2a1993c0b402 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 5 Jul 2022 15:01:32 +0200 Subject: [PATCH 05/34] ~ updated django-cas-ng to 4.3.0 --- src/requirements-dev.txt | 2 +- src/requirements.in | 2 +- src/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index c64d099c..ae89ba2e 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -40,7 +40,7 @@ deprecated==1.2.13 # via redis distlib==0.3.4 # via virtualenv -django-cas-ng==4.1.1 +django-cas-ng==4.3.0 # via -r src/requirements.in django-cleanup==5.2.0 # via -r src/requirements.in diff --git a/src/requirements.in b/src/requirements.in index 50074bd7..a3e2d274 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -7,7 +7,7 @@ apispec==0.39.0 bibtexparser==1.2.0 concurrent-log-handler==0.9.19 django~=2.2 -django-cas-ng==4.1.1 +django-cas-ng==4.3.0 django-cleanup==5.2.0 django-cors-headers==3.7.0 django-debug-toolbar==3.2.1 diff --git a/src/requirements.txt b/src/requirements.txt index 0c03ebd2..3d9960a9 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -36,7 +36,7 @@ croniter==1.3.4 # via rq-scheduler deprecated==1.2.13 # via redis -django-cas-ng==4.1.1 +django-cas-ng==4.3.0 # via -r src/requirements.in django-cleanup==5.2.0 # via -r src/requirements.in From ff53070b05140968eb9b4e08ed813e37520bbc41 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 5 Jul 2022 16:11:42 +0200 Subject: [PATCH 06/34] ~ using pre-commit base-hooks --- .pre-commit-config.yaml | 46 +++-------------------------------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad7a4e9c..6439e67e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,47 +2,7 @@ default_language_version: python: python3 exclude: (/migrations/|manage.py|docs/source/conf.py) repos: - - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 + - repo: https://github.com/base-angewandte/pre-commit-hooks + rev: py3.7 hooks: - - id: pyupgrade - args: [--py37-plus] - - repo: https://github.com/timothycrosley/isort - rev: 5.10.1 - hooks: - - id: isort - - repo: https://github.com/psf/black - rev: 21.12b0 - hooks: - - id: black - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 - hooks: - - id: double-quote-string-fixer - - id: end-of-file-fixer - - id: check-yaml - - repo: https://github.com/myint/docformatter - rev: v1.4 - hooks: - - id: docformatter - args: [--in-place] - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - additional_dependencies: [flake8-bugbear==22.1.11, pep8-naming==0.12.1] - - repo: https://github.com/PyCQA/bandit - rev: 1.7.1 - hooks: - - id: bandit - exclude: tests.py - - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.0.1 - hooks: - - id: docker-compose-check - - id: hadolint - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.1 - hooks: - - id: prettier - types_or: [markdown, yaml] + - id: base-hooks From 7862efb8b578fa3f5efea7e7a934adf65725434b Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 5 Jul 2022 16:12:51 +0200 Subject: [PATCH 07/34] + added CAS_CHECK_NEXT environment variable for development --- CHANGELOG.md | 1 + docs/source/configuration.md | 8 +++++++- src/portfolio/env-skel | 1 + src/portfolio/settings.py | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97ef1318..58c72053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Send 301 for retrieve requests in EntryViewSet with old entry ids +- Added `CAS_CHECK_NEXT` environment variable for development ### Changed diff --git a/docs/source/configuration.md b/docs/source/configuration.md index 31ea33c7..5a71fbab 100644 --- a/docs/source/configuration.md +++ b/docs/source/configuration.md @@ -78,6 +78,12 @@ Default: `True` If you are running Portfolio behind nginx, this has to be `True`. For local development, set this to `False`. +### `CAS_CHECK_NEXT` + +Default: `True` + +If you want to use a non-local `CAS_REDIRECT_URL`, e.g. when redirecting to the frontend in a development setup, set this to `True`. + ### `CAS_REDIRECT_URL` Default: `FORCE_SCRIPT_NAME` if defined, else `/` @@ -399,7 +405,7 @@ a local dev environment it might rather be something like Default: `None` -The API key for this Portfolio instance, as it is defined in the Showroom admin. +The API key for this Portfolio instance, as it is defined in the Showroom admin. #### `SHOWROOM_REPO_ID` diff --git a/src/portfolio/env-skel b/src/portfolio/env-skel index 09c07bc7..cd6c6647 100644 --- a/src/portfolio/env-skel +++ b/src/portfolio/env-skel @@ -12,6 +12,7 @@ SITE_URL=https://url/ # CAS_VERSION=3 # CAS_SERVER= # CAS_REDIRECT_URL= +# CAS_CHECK_NEXT=True # CAS_VERIFY_CERTIFICATE=True # CAS_RENAME_ATTRIBUTES= # EMAIL_HOST_USER= diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index 16f9dc34..a2fe4d9c 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -139,6 +139,7 @@ CAS_VERSION = env.str('CAS_VERSION', default='3') CAS_APPLY_ATTRIBUTES_TO_USER = True CAS_REDIRECT_URL = env.str('CAS_REDIRECT_URL', default=FORCE_SCRIPT_NAME or '/') +CAS_CHECK_NEXT = env.bool('CAS_CHECK_NEXT', default=True) CAS_VERIFY_CERTIFICATE = env.bool('CAS_VERIFY_CERTIFICATE', default=True) CAS_RENAME_ATTRIBUTES = env.dict('CAS_RENAME_ATTRIBUTES', default={}) From 61cc669590001aff32251b5cc583ea0d2b0ea592 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 11 Jul 2022 16:00:28 +0200 Subject: [PATCH 08/34] + added mixins + added CreateListMixin --- src/api/mixins.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/api/mixins.py diff --git a/src/api/mixins.py b/src/api/mixins.py new file mode 100644 index 00000000..dae0cd55 --- /dev/null +++ b/src/api/mixins.py @@ -0,0 +1,8 @@ +class CreateListMixin: + """Allows bulk creation of a resource.""" + + def get_serializer(self, *args, **kwargs): + if isinstance(kwargs.get('data', {}), list): + kwargs['many'] = True + + return super().get_serializer(*args, **kwargs) From d783b1064ae487a4c9a21a86a4e6364c43e91832 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 11 Jul 2022 16:01:22 +0200 Subject: [PATCH 09/34] ~ moved CountModelMixin to mixins --- src/api/mixins.py | 17 +++++++++++++++++ src/api/views.py | 12 +----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/api/mixins.py b/src/api/mixins.py index dae0cd55..b5d57901 100644 --- a/src/api/mixins.py +++ b/src/api/mixins.py @@ -1,3 +1,20 @@ +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework.decorators import action +from rest_framework.response import Response + + +class CountModelMixin: + """Count a queryset.""" + + @swagger_auto_schema(manual_parameters=[], responses={200: openapi.Response('')}) + @action(detail=False, filter_backends=[], pagination_class=None) + def count(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + content = {'count': queryset.count()} + return Response(content) + + class CreateListMixin: """Allows bulk creation of a resource.""" diff --git a/src/api/views.py b/src/api/views.py index 6746e9cd..829729f8 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -40,6 +40,7 @@ from media_server.models import get_media_for_entry, update_media_order_for_entry from media_server.utils import get_free_space_for_user +from .mixins import CountModelMixin from .serializers.entry import EntrySerializer from .serializers.relation import RelationSerializer from .yasg import ( @@ -84,17 +85,6 @@ class StandardLimitOffsetPagination(LimitOffsetPagination): # offset_query_param = 'skip' -class CountModelMixin: - """Count a queryset.""" - - @swagger_auto_schema(manual_parameters=[], responses={200: openapi.Response('')}) - @action(detail=False, filter_backends=[], pagination_class=None) - def count(self, request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - content = {'count': queryset.count()} - return Response(content) - - class EntryFilter(FilterSet): type = CharFilter(field_name='type', lookup_expr='source__iexact') From 9181e6d081ab7f13471223e85e6a2281b25e4f3e Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 11 Jul 2022 16:06:05 +0200 Subject: [PATCH 10/34] ~ using CreateListMixin --- src/api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/views.py b/src/api/views.py index 829729f8..bf124d68 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -40,7 +40,7 @@ from media_server.models import get_media_for_entry, update_media_order_for_entry from media_server.utils import get_free_space_for_user -from .mixins import CountModelMixin +from .mixins import CountModelMixin, CreateListMixin from .serializers.entry import EntrySerializer from .serializers.relation import RelationSerializer from .yasg import ( @@ -121,7 +121,7 @@ class Meta: ), name='list', ) -class EntryViewSet(viewsets.ModelViewSet, CountModelMixin): +class EntryViewSet(CreateListMixin, viewsets.ModelViewSet, CountModelMixin): """ retrieve: Returns a certain entry. From fab918bc4d876d9a348a2d978cf010d272883de2 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 11 Jul 2022 16:16:03 +0200 Subject: [PATCH 11/34] + added EntryBulkCreateSerializer --- src/api/serializers/entry.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/api/serializers/entry.py b/src/api/serializers/entry.py index 477a8f4a..cc75d4b8 100644 --- a/src/api/serializers/entry.py +++ b/src/api/serializers/entry.py @@ -3,6 +3,7 @@ from drf_yasg.utils import swagger_serializer_method from rest_framework import serializers +from django.db.models.signals import post_save from django.urls import reverse_lazy from django.utils.text import format_lazy from django.utils.translation import gettext_lazy as _ @@ -44,6 +45,17 @@ class RelationsSerializer(serializers.Serializer): to = RelatedEntrySerializer(read_only=True) +class EntryBulkCreateSerializer(serializers.ListSerializer): + def create(self, validated_data): + entries_data = [Entry(**data) for data in validated_data] + entries = Entry.objects.bulk_create(entries_data) + # send post_save signal for published entries so they are pushed to Showroom + for entry in entries: + if entry.published: + post_save.send(Entry, instance=entry, created=True) + return entries + + class EntrySerializer(CleanModelSerializer, SwaggerMetaModelSerializer): owner = serializers.HiddenField(default=serializers.CurrentUserDefault()) parents = serializers.SerializerMethodField() @@ -53,6 +65,7 @@ class EntrySerializer(CleanModelSerializer, SwaggerMetaModelSerializer): class Meta: model = Entry + list_serializer_class = EntryBulkCreateSerializer fields = '__all__' swagger_meta_attrs = { 'id': OrderedDict([('hidden', True)]), From ae5ff4fb60598f2b0f3ba237cffa4907c5063f86 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 19 Jul 2022 09:55:42 +0200 Subject: [PATCH 12/34] + added first_name and last_name to user_information response --- src/api/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/views.py b/src/api/views.py index 6746e9cd..641dfebb 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -332,6 +332,8 @@ def user_information(request, *args, **kwargs): 'uuid': request.user.username, 'name': attributes.get('display_name'), 'email': attributes.get('email'), + 'first_name': request.user.first_name, + 'last_name': request.user.last_name, 'permissions': attributes.get('permissions') or [], 'groups': attributes.get('groups') or [], 'space': get_free_space_for_user(request.user) if request.user else None, From 0692ccb47e46b0bbf752c3191311bc8d493c1881 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 19 Jul 2022 09:56:30 +0200 Subject: [PATCH 13/34] ~ using request.user.get_full_name() and request.user.email to be consistent over Portfolio instances --- src/api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/views.py b/src/api/views.py index 641dfebb..a46faaa1 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -330,10 +330,10 @@ def user_information(request, *args, **kwargs): attributes = request.session.get('attributes', {}) data = { 'uuid': request.user.username, - 'name': attributes.get('display_name'), - 'email': attributes.get('email'), + 'name': request.user.get_full_name(), 'first_name': request.user.first_name, 'last_name': request.user.last_name, + 'email': request.user.email, 'permissions': attributes.get('permissions') or [], 'groups': attributes.get('groups') or [], 'space': get_free_space_for_user(request.user) if request.user else None, From 0497ded822cc02935b1a2914f7a219bf2860d51e Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 19 Jul 2022 09:57:06 +0200 Subject: [PATCH 14/34] ~ CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c72053..6959dc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,14 @@ - Send 301 for retrieve requests in EntryViewSet with old entry ids - Added `CAS_CHECK_NEXT` environment variable for development +- Added `first_name` and `last_name` to API user response ### Changed - **BREAKING**: Updated shortuuid to 1.0.1 - **BREAKING**: Migrated all existing shortuuids to new format - **BREAKING**: Migrated all existing media directories +- **BREAKING**: Adapted API user response to use `request.user.get_full_name()` to be consistent over Portfolio instances ## 1.1.6 From 6c65322998e316e0766fa06879a928eb4a1c565f Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 11:58:09 +0200 Subject: [PATCH 15/34] ~ bugfix fix_keywords --- src/core/management/commands/fix_keywords.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/management/commands/fix_keywords.py b/src/core/management/commands/fix_keywords.py index 722560ac..d70627d5 100644 --- a/src/core/management/commands/fix_keywords.py +++ b/src/core/management/commands/fix_keywords.py @@ -1,7 +1,6 @@ from progressbar import progressbar from titlecase import titlecase -from django.conf import settings from django.core.management.base import BaseCommand from core.models import Entry @@ -24,7 +23,7 @@ def handle(self, *args, **options): if 'disciplines' in kw['source']: project = 'disciplines' else: - project = settings.VOC_ID + project = 'basekw' e.keywords[idx]['label'] = { 'de': get_preflabel(concept, project=project, graph=f'{graph}/', lang='de'), 'en': titlecase(get_preflabel(concept, project=project, graph=f'{graph}/', lang='en')), From 9a3a842c74a962ef3073834c62376d63568fdb0b Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:00:40 +0200 Subject: [PATCH 16/34] + added keywords_usage management command --- .../management/commands/keywords_usage.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 src/core/management/commands/keywords_usage.py diff --git a/src/core/management/commands/keywords_usage.py b/src/core/management/commands/keywords_usage.py new file mode 100644 index 00000000..e5f97fe0 --- /dev/null +++ b/src/core/management/commands/keywords_usage.py @@ -0,0 +1,69 @@ +import csv +from collections import Counter +from datetime import date + +from progressbar import progressbar +from titlecase import titlecase + +from django.core.management.base import BaseCommand + +from core.models import Entry +from core.skosmos import get_preflabel + + +class Command(BaseCommand): + help = 'Return usage of keywords of published entries' + + def handle(self, *args, **options): + keywords_list = [] + for e in Entry.objects.filter(published=True): + if e.keywords: + for kw in e.keywords: + if kw.get('source'): + keywords_list.append(kw['source']) + else: + keywords_list.append(kw['label']['en']) + + counter = Counter(keywords_list) + + today = date.today().strftime('%d-%m-%Y') + + with open(f'export/{today}_keywords_usage.csv', mode='w') as csvfile: + csv_writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + csv_writer.writerow( + [ + 'count', + 'source', + 'label_en', + 'label_de', + ] + ) + for kw, num in progressbar(counter.most_common()): + if kw.startswith('http'): + graph, concept = kw.rsplit('/', 1) + if 'disciplines' in kw: + project = 'disciplines' + else: + project = 'basekw' + label_en = titlecase(get_preflabel(concept, project=project, graph=f'{graph}/', lang='en')) + label_de = get_preflabel(concept, project=project, graph=f'{graph}/', lang='de') + + csv_writer.writerow( + [ + num, + kw, + label_en, + label_de, + ] + ) + else: + csv_writer.writerow( + [ + num, + '', + kw, + kw, + ] + ) + + self.stdout.write(self.style.SUCCESS('Successfully exported keywords usage')) From 22c8fbd6ed6bf688a28434d8cd150cc28a731882 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:02:48 +0200 Subject: [PATCH 17/34] + added project mapping --- src/core/skosmos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/core/skosmos.py b/src/core/skosmos.py index 9a497bb2..092976bd 100644 --- a/src/core/skosmos.py +++ b/src/core/skosmos.py @@ -14,6 +14,16 @@ CACHE_TIME = 86400 # 1 day +PROJECT_MAPPING = { + 'http://base.uni-ak.ac.at/portfolio/languages/': 'languages', + 'http://base.uni-ak.ac.at/recherche/keywords/': 'basekw', + 'http://base.uni-ak.ac.at/vocabulary/': 'basevoc', + 'http://base.uni-ak.ac.at/portfolio/licenses/': 'licenses', + 'http://base.uni-ak.ac.at/portfolio/taxonomy/': 'potax', + 'http://base.uni-ak.ac.at/portfolio/vocabulary/': 'povoc', + 'http://base.uni-ak.ac.at/portfolio/disciplines/': 'disciplines', +} + skosmos = SkosmosClient(api_base=settings.SKOSMOS_API) From 52071a9fdd104db389ab646e96bdf6ae4908f695 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:04:06 +0200 Subject: [PATCH 18/34] + added get_preflabel_via_uri --- src/core/skosmos.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core/skosmos.py b/src/core/skosmos.py index 092976bd..f25fce5b 100644 --- a/src/core/skosmos.py +++ b/src/core/skosmos.py @@ -286,6 +286,12 @@ def get_preflabel(concept, project=settings.VOC_ID, graph=settings.VOC_GRAPH, la return label or '' +def get_preflabel_via_uri(concept, lang=None): + g, c = concept.rsplit('/', 1) + g += '/' + return get_preflabel(c, project=PROJECT_MAPPING[g], graph=g, lang=lang) + + def get_collection_members(collection, maxhits=1000, use_cache=True): cache_key = f'get_collection_members_{collection}' From bdecd0e6d283cae9ba6843abdf564d66080d582f Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:05:05 +0200 Subject: [PATCH 19/34] + added EN_LABELS_TITLE_CASE setting --- src/portfolio/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index a2fe4d9c..2fdcd235 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -423,6 +423,8 @@ VOC_GRAPH = 'http://base.uni-ak.ac.at/portfolio/vocabulary/' LANGUAGES_VOCID = 'languages' +EN_LABELS_TITLE_CASE = env.bool('EN_LABELS_TITLE_CASE', default=True) + ANGEWANDTE_API_KEY = env.str('ANGEWANDTE_API_KEY', default='') GEONAMES_USER = env.str('GEONAMES_USER', default=None) PELIAS_API_KEY = env.str('PELIAS_API_KEY', default=None) From fd93aa874fe1cc7a4baef2b8356f9e7d9728774e Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:09:31 +0200 Subject: [PATCH 20/34] + added update_labels management command --- src/core/management/commands/update_labels.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/core/management/commands/update_labels.py diff --git a/src/core/management/commands/update_labels.py b/src/core/management/commands/update_labels.py new file mode 100644 index 00000000..80371d9d --- /dev/null +++ b/src/core/management/commands/update_labels.py @@ -0,0 +1,57 @@ +from progressbar import progressbar +from titlecase import titlecase + +from django.conf import settings +from django.contrib.postgres.fields import JSONField +from django.core.management.base import BaseCommand + +from core.models import Entry +from core.skosmos import get_preflabel_via_uri + + +class Command(BaseCommand): + help = 'Update labels in case they have been changed in the vocabulary' + + # helper variable to prevent unnecessary savings + _need_to_save = False + + # helper function for walking through the JSON + def _walk(self, obj): + if isinstance(obj, list): + for list_item in obj: + self._walk(list_item) + elif isinstance(obj, dict): + if 'source' in obj: + if obj['source'].startswith('http://base'): + label_de = get_preflabel_via_uri(obj['source'], lang='de') + label_en = get_preflabel_via_uri(obj['source'], lang='en') + if settings.EN_LABELS_TITLE_CASE: + label_en = titlecase(label_en) + if 'label' not in obj: + obj['label'] = {} + if obj['label'].get('de') != label_de: + obj['label']['de'] = label_de + self._need_to_save = True + if obj['label'].get('en') != label_en: + obj['label']['en'] = label_en + self._need_to_save = True + if 'roles' in obj: + self._walk(obj['roles']) + else: + for _k, v in obj.items(): + self._walk(v) + + def handle(self, *args, **options): + # gather all fields containing concepts + fields_to_update = [] + model_fields = Entry._meta.fields + for field in model_fields: + if isinstance(field, JSONField): + fields_to_update.append(field.name) + + for e in progressbar(Entry.objects.all()): + for field in fields_to_update: + self._walk(getattr(e, field)) + if self._need_to_save: + e.save() + self._need_to_save = False From 8b96c5e8f19ea3d7330ca60d2e50c0fd395f06a2 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:42:11 +0200 Subject: [PATCH 21/34] + added update-labels command to Makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 634462a9..abe08f12 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,9 @@ restart-gunicorn: restart-rq: docker-compose restart portfolio-rq-worker-1 portfolio-rq-worker-2 portfolio-rq-worker-3 portfolio-rq-scheduler +update-labels: + docker-compose exec portfolio-django python manage.py update_labels + update: git-update init init-rq init-static restart-gunicorn restart-rq build-docs start-dev: From bbe9677827621617c5e9bf5d880e5fef7e470df9 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Wed, 7 Sep 2022 12:42:51 +0200 Subject: [PATCH 22/34] + run update-labels on every update --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index abe08f12..26c9bb3d 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ restart-rq: update-labels: docker-compose exec portfolio-django python manage.py update_labels -update: git-update init init-rq init-static restart-gunicorn restart-rq build-docs +update: git-update init init-rq init-static restart-gunicorn restart-rq build-docs update-labels start-dev: docker-compose pull --ignore-pull-failures From a4cda5c2ea925e28aed10da7816270f7dc9765c3 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Tue, 11 Oct 2022 15:43:30 +0200 Subject: [PATCH 23/34] ~ cherry picked 1860db55f58cc5d7c6ac0c169e35e096d89d08cb --- src/portfolio/settings.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index 47100bf0..a891cde3 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -425,6 +425,8 @@ EN_LABELS_TITLE_CASE = env.bool('EN_LABELS_TITLE_CASE', default=True) +PRIMO_API_URL = 'https://apigw.obvsg.at/primo/v1/search' +PRIMO_API_KEY = env.str('PORTFOLIO_AUTOSUGGEST_PRIMOAPIKEY', default='') ANGEWANDTE_API_KEY = env.str('ANGEWANDTE_API_KEY', default='') GEONAMES_USER = env.str('GEONAMES_USER', default=None) PELIAS_API_KEY = env.str('PELIAS_API_KEY', default=None) @@ -560,6 +562,13 @@ ), }, }, + 'PRIMO_IMPORT': { + apiconfig.URL: PRIMO_API_URL, + apiconfig.QUERY_FIELD: 'q', + apiconfig.PAYLOAD: {'vid': 'OBV', 'scope': 'OBV_Gesamt'}, + apiconfig.TIMEOUT: 10, + apiconfig.HEADER: {'apikey': PRIMO_API_KEY}, + }, } ANGEWANDTE_MAPPING = { @@ -602,6 +611,15 @@ 'geometry': ('geometry',), } +PRIMO_MAPPING = { + 'source': ('pnx', 'search', 'lsr33'), + 'label': ('pnx', 'search', 'title'), + 'isbn': ('pnx', 'search', 'isbn'), + 'description': ('pnx', 'search', 'description'), + 'author': ('pnx', 'display', 'creator'), + 'pages': ('pnx', 'display', 'format'), +} + RESPONSE_MAPS = { 'ANGEWANDTE_PERSON': { @@ -687,8 +705,14 @@ }, }, }, + 'PRIMO_IMPORT': { + apiconfig.RESULT: 'docs', + apiconfig.DIRECT: PRIMO_MAPPING, + apiconfig.RULES: {'source_name': {apiconfig.RULE: '"OBVSG"'}}, + }, } +BIBRECS = ('PRIMO_IMPORT',) CONTRIBUTORS = ('GND_PERSON', 'GND_INSTITUTION', 'VIAF_PERSON', 'VIAF_INSTITUTION') if 'uni-ak.ac.at' in SITE_URL: @@ -709,6 +733,7 @@ ACTIVE_SOURCES = { 'contributors': CONTRIBUTORS, 'places': PLACES, + 'bibrecs': BIBRECS, 'keywords': {'all': 'core.skosmos.get_base_keywords', 'search': 'core.skosmos.get_keywords'}, 'roles': 'core.skosmos.get_roles', 'formats': 'core.skosmos.get_formats', From 3cab6ca8c0c2be19e082783b519d9e534e50b5d5 Mon Sep 17 00:00:00 2001 From: Roman T Date: Mon, 25 Apr 2022 11:50:31 +0300 Subject: [PATCH 24/34] + more mapping fields for the primo import feature, and set results limit=100 (cherry picked from commit 6733198fd36b73a8e94713e475339e53efec3fcd) --- src/portfolio/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index a891cde3..d0839146 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -565,7 +565,7 @@ 'PRIMO_IMPORT': { apiconfig.URL: PRIMO_API_URL, apiconfig.QUERY_FIELD: 'q', - apiconfig.PAYLOAD: {'vid': 'OBV', 'scope': 'OBV_Gesamt'}, + apiconfig.PAYLOAD: {'vid': 'OBV', 'scope': 'OBV_Gesamt', 'limit': 100}, apiconfig.TIMEOUT: 10, apiconfig.HEADER: {'apikey': PRIMO_API_KEY}, }, @@ -618,6 +618,11 @@ 'description': ('pnx', 'search', 'description'), 'author': ('pnx', 'display', 'creator'), 'pages': ('pnx', 'display', 'format'), + 'creationdate': ('pnx', 'display', 'creationdate'), + 'type': ('pnx', 'display', 'type'), + 'lad24': ('pnx', 'addata', 'lad24'), + 'language': ('pnx', 'display', 'language'), + 'subject': ('pnx', 'display', 'subject'), } From dd06d71cdb476afc79639e3b3e425df8be560e4f Mon Sep 17 00:00:00 2001 From: Roman T Date: Wed, 25 May 2022 16:48:26 +0300 Subject: [PATCH 25/34] + two more mapping fields (contributor, ispartof) (cherry picked from commit 07692b07362a800886ef6f4d7c9dec144d7a6c9d) --- src/portfolio/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index d0839146..ec656acd 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -623,6 +623,8 @@ 'lad24': ('pnx', 'addata', 'lad24'), 'language': ('pnx', 'display', 'language'), 'subject': ('pnx', 'display', 'subject'), + 'contributors': ('pnx', 'display', 'contributor'), + 'ispartof': ('pnx', 'display', 'ispartof'), } From c26b7924deadc8c62f957f8a6eb096184062640e Mon Sep 17 00:00:00 2001 From: Roman Turcanu Date: Tue, 31 May 2022 15:40:42 +0300 Subject: [PATCH 26/34] feat. #810: + add the lds16 field to primo mappings (cherry picked from commit 823591528712567908a994d3f871b53e991d1079) --- src/portfolio/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index ec656acd..157c6167 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -621,6 +621,7 @@ 'creationdate': ('pnx', 'display', 'creationdate'), 'type': ('pnx', 'display', 'type'), 'lad24': ('pnx', 'addata', 'lad24'), + 'lds16': ('pnx', 'display', 'lds16'), 'language': ('pnx', 'display', 'language'), 'subject': ('pnx', 'display', 'subject'), 'contributors': ('pnx', 'display', 'contributor'), From 61a8f3b8bf1daefa79a3316419f5cf03d8d7a037 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Fri, 21 Oct 2022 12:17:36 +0200 Subject: [PATCH 27/34] fix(media_server): add missing migration --- .../migrations/0022_auto_20221021_1215.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/media_server/migrations/0022_auto_20221021_1215.py diff --git a/src/media_server/migrations/0022_auto_20221021_1215.py b/src/media_server/migrations/0022_auto_20221021_1215.py new file mode 100644 index 00000000..a0c47c4d --- /dev/null +++ b/src/media_server/migrations/0022_auto_20221021_1215.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.28 on 2022-10-21 10:15 + +from django.db import migrations, models +import media_server.clamav +import media_server.models +import media_server.storages + + +class Migration(migrations.Migration): + + dependencies = [ + ('media_server', '0021_media_featured'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='file', + field=models.FileField(max_length=255, storage=media_server.storages.ProtectedFileSystemStorage(), upload_to=media_server.models.user_directory_path, validators=[media_server.clamav.validate_file_infection]), + ), + ] From 57720413a7138521a0ec329643e2b456c883f681 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Fri, 21 Oct 2022 13:06:37 +0200 Subject: [PATCH 28/34] fix(locale): add missing translation --- src/locale/de/LC_MESSAGES/django.mo | Bin 2566 -> 2660 bytes src/locale/de/LC_MESSAGES/django.po | 41 ++++++++++++++++------------ src/locale/en/LC_MESSAGES/django.po | 41 ++++++++++++++++------------ 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/locale/de/LC_MESSAGES/django.mo b/src/locale/de/LC_MESSAGES/django.mo index 10ca8f7f8a7f47d9560746492296fcf87e9a22d5..12b68570ac7bc3c3ba0ae36d8925d336540047fd 100644 GIT binary patch delta 738 zcmY+?%_~Gv7{~F)m@$mE!FU^nEAqbbR*KCKS@;iZCa&Bv5;qeYl4Oapla=xZOk`ze z#m-*QER}_ng@y0WOv+PdKIhzf?m6dq&K$;avEAp|;F%E#S}QGBWY&jos5lOob>j)@ z`E6nRiUW+xip_>_6l-t=!?=UhxQ{VBMeT8gC3ugGX1n%Cr-cVO)CU$ci$fHfZ~&QT z)7Xg%sQG)?faitxS?po_fL-{3+OR2PHjZOhhwI2zc7XNlZ%1^JWP5?y;DzH2V~8{+ zalBv#lZ>yi3cs)v!~AIBDC(eH*p3s(6k9+YcnP)s9;#B8*vkHPQ<(6C%KRNWFjAJk zl0h6~Jcn(#g}r!&D(xL={u7SiC#rJ&k$k38sH>et-D)LG2c|&EKc9^@)?&H>WkE40 z&;Q{2ztDoUv}T&JQ$~skn(kLuqu?yR#~Ou3fefAm{MJyayJ=|H?FG2^Ye zj<@M}Ys>DU?=CqR&tGxUsntws!_6KRZwC`8-}RW{J892%=JRhf-`nxrjqGy)T1N6vy$?7j02*)lx;{{UuB#kq!nO8u1s1gftono3~=?#zbNe5~IXqYb?el zA`&rl6fq%b#4aJ?`>V&`r04lOH_z?8=ic)i-VNuULf#`II-LB%e)7@9gIIN&4d5CM z%}7j1Axd1H8kllbYFSofYTb@OI&tZ)H?U0vlyl^Jm<0#`djAOLg zYzQY%SD8W0&*3;8qi*aDb;1|aMrBkNwV4X2B^4@`r7Mxw9~(*aM%C+tI#nInMruJW z$Y`_Gkbj{S&{eqh?05_F0l)hP DOj9>i diff --git a/src/locale/de/LC_MESSAGES/django.po b/src/locale/de/LC_MESSAGES/django.po index 62a09715..15eefc8f 100644 --- a/src/locale/de/LC_MESSAGES/django.po +++ b/src/locale/de/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-03-18 16:44+0100\n" +"POT-Creation-Date: 2022-10-21 13:01+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -17,7 +17,7 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" -#: api/serializers/entry.py:125 +#: api/serializers/entry.py:139 msgid "will not be published" msgstr "Werden nicht veröffentlicht" @@ -25,38 +25,38 @@ msgstr "Werden nicht veröffentlicht" msgid "Relation already exists" msgstr "Verknüpfung existiert bereits" -#: api/serializers/relation.py:30 api/views.py:187 +#: api/serializers/relation.py:30 api/views.py:195 api/views.py:225 msgid "Current user is not the owner of this entry" msgstr "Der momentane Benutzer ist nicht der Besitzer dieses Eintrags" -#: api/views.py:191 api/views.py:776 api/views.py:801 -#: media_server/serializers.py:30 +#: api/views.py:199 api/views.py:232 api/views.py:1048 api/views.py:1077 +#: media_server/serializers.py:38 msgid "Entry does not exist" msgstr "Eintrag existiert nicht" -#: api/views.py:272 +#: api/views.py:313 msgid "Schema not found for the given query" msgstr "Kein Schema gefunden" -#: api/views.py:311 api/views.py:771 +#: api/views.py:354 api/views.py:1043 msgid "User does not exist" msgstr "Benutzer existiert nicht" -#: core/models.py:119 +#: core/models.py:121 #, python-format msgid "Type %(type_source)s does not belong to any active schema" msgstr "Typ %(type_source)s gehört zu keinem aktiven Schema" -#: core/models.py:126 +#: core/models.py:128 #, python-format msgid "Invalid data: %(error)s" msgstr "Ungültige Daten: %(error)s" -#: core/models.py:129 +#: core/models.py:131 msgid "Data without type" msgstr "Daten ohne Typ" -#: core/models.py:165 +#: core/models.py:167 msgid "Both entries must belong to the same user" msgstr "Beide Einträge müssen demselben Benutzer gehören" @@ -103,7 +103,11 @@ msgstr "Dieselbe Sprache wird mehrmals definiert" msgid "Invalid texts: %(error)s" msgstr "Ungültige Texte: %(error)s" -#: media_server/serializers.py:33 +#: media_server/clamav.py:36 +msgid "File is infected with malware" +msgstr "Datei ist mit Malware infiziert" + +#: media_server/serializers.py:41 msgid "Current user is not the owner of entry" msgstr "Der momentane Benutzer ist nicht der Besitzer dieses Eintrags" @@ -116,23 +120,24 @@ msgstr "Ungültige Lizenz: %(error)s" msgid "Invalid license" msgstr "Ungültige Lizenz" -#: media_server/views.py:98 media_server/views.py:179 media_server/views.py:224 +#: media_server/views.py:115 media_server/views.py:201 +#: media_server/views.py:246 msgid "Current user is not the owner of this media object" msgstr "Der momentane Benutzer ist nicht der Besitzer dieses Medienobjekts" -#: media_server/views.py:111 media_server/views.py:187 -#: media_server/views.py:232 +#: media_server/views.py:133 media_server/views.py:209 +#: media_server/views.py:254 msgid "Media object does not exist" msgstr "Medienobjekt existiert nicht" -#: media_server/views.py:129 +#: media_server/views.py:151 msgid "No space left for user" msgstr "Kein Speicherplatz mehr für diesen Benutzer" -#: portfolio/settings.py:247 +#: portfolio/settings.py:250 msgid "German" msgstr "Deutsch" -#: portfolio/settings.py:248 +#: portfolio/settings.py:251 msgid "English" msgstr "Englisch" diff --git a/src/locale/en/LC_MESSAGES/django.po b/src/locale/en/LC_MESSAGES/django.po index 5858fd9f..5f740537 100644 --- a/src/locale/en/LC_MESSAGES/django.po +++ b/src/locale/en/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-03-18 16:44+0100\n" +"POT-Creation-Date: 2022-10-21 13:03+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -16,7 +16,7 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -#: api/serializers/entry.py:125 +#: api/serializers/entry.py:139 msgid "will not be published" msgstr "" @@ -24,38 +24,38 @@ msgstr "" msgid "Relation already exists" msgstr "" -#: api/serializers/relation.py:30 api/views.py:187 +#: api/serializers/relation.py:30 api/views.py:195 api/views.py:225 msgid "Current user is not the owner of this entry" msgstr "" -#: api/views.py:191 api/views.py:776 api/views.py:801 -#: media_server/serializers.py:30 +#: api/views.py:199 api/views.py:232 api/views.py:1048 api/views.py:1077 +#: media_server/serializers.py:38 msgid "Entry does not exist" msgstr "" -#: api/views.py:272 +#: api/views.py:313 msgid "Schema not found for the given query" msgstr "" -#: api/views.py:311 api/views.py:771 +#: api/views.py:354 api/views.py:1043 msgid "User does not exist" msgstr "" -#: core/models.py:119 +#: core/models.py:121 #, python-format msgid "Type %(type_source)s does not belong to any active schema" msgstr "" -#: core/models.py:126 +#: core/models.py:128 #, python-format msgid "Invalid data: %(error)s" msgstr "" -#: core/models.py:129 +#: core/models.py:131 msgid "Data without type" msgstr "" -#: core/models.py:165 +#: core/models.py:167 msgid "Both entries must belong to the same user" msgstr "" @@ -102,7 +102,11 @@ msgstr "" msgid "Invalid texts: %(error)s" msgstr "" -#: media_server/serializers.py:33 +#: media_server/clamav.py:36 +msgid "File is infected with malware" +msgstr "" + +#: media_server/serializers.py:41 msgid "Current user is not the owner of entry" msgstr "" @@ -115,23 +119,24 @@ msgstr "" msgid "Invalid license" msgstr "" -#: media_server/views.py:98 media_server/views.py:179 media_server/views.py:224 +#: media_server/views.py:115 media_server/views.py:201 +#: media_server/views.py:246 msgid "Current user is not the owner of this media object" msgstr "" -#: media_server/views.py:111 media_server/views.py:187 -#: media_server/views.py:232 +#: media_server/views.py:133 media_server/views.py:209 +#: media_server/views.py:254 msgid "Media object does not exist" msgstr "" -#: media_server/views.py:129 +#: media_server/views.py:151 msgid "No space left for user" msgstr "" -#: portfolio/settings.py:247 +#: portfolio/settings.py:250 msgid "German" msgstr "" -#: portfolio/settings.py:248 +#: portfolio/settings.py:251 msgid "English" msgstr "" From 2a5f6e2d783b0ce2ee5f9d89c345752d02f75252 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Mon, 24 Oct 2022 12:20:31 +0200 Subject: [PATCH 29/34] build(deps): upgrade pip-tools to 6.9.0 --- src/requirements-dev.txt | 14 +++++++++++--- src/requirements.in | 2 +- src/requirements.txt | 14 +++++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/requirements-dev.txt b/src/requirements-dev.txt index a74a6e69..0a099d00 100644 --- a/src/requirements-dev.txt +++ b/src/requirements-dev.txt @@ -14,6 +14,8 @@ attrs==21.4.0 # via jsonschema bibtexparser==1.2.0 # via -r src/requirements.in +build==0.8.0 + # via pip-tools certifi==2021.10.8 # via # requests @@ -94,6 +96,7 @@ idna==3.3 # yarl importlib-metadata==4.11.3 # via + # build # click # jsonschema # markdown @@ -131,11 +134,12 @@ nodeenv==1.6.0 # via pre-commit packaging==21.3 # via + # build # drf-yasg # redis pep517==0.12.0 - # via pip-tools -pip-tools==6.1.0 + # via build +pip-tools==6.9.0 # via -r src/requirements.in platformdirs==2.5.1 # via virtualenv @@ -234,7 +238,9 @@ titlecase==2.3 toml==0.10.2 # via pre-commit tomli==2.0.1 - # via pep517 + # via + # build + # pep517 typing-extensions==4.1.1 # via # async-timeout @@ -257,6 +263,8 @@ webcolors==1.11.1 # via jsonschema werkzeug==1.0.1 # via -r src/requirements.in +wheel==0.37.1 + # via pip-tools wrapt==1.14.0 # via # deprecated diff --git a/src/requirements.in b/src/requirements.in index f2893e06..6ee75d7e 100644 --- a/src/requirements.in +++ b/src/requirements.in @@ -23,7 +23,7 @@ hashids==1.3.1 jsonschema[format]==3.2.0 markdown==3.3.4 # for djangorestframework marshmallow==2.21.0 -pip-tools==6.1.0 +pip-tools==6.9.0 progressbar2==3.53.1 psycopg2-binary==2.8.6 python-magic==0.4.22 diff --git a/src/requirements.txt b/src/requirements.txt index 61a42bc3..272a915e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -14,6 +14,8 @@ attrs==21.4.0 # via jsonschema bibtexparser==1.2.0 # via -r src/requirements.in +build==0.8.0 + # via pip-tools certifi==2021.10.8 # via # requests @@ -85,6 +87,7 @@ idna==3.3 # requests importlib-metadata==4.11.3 # via + # build # click # jsonschema # markdown @@ -116,11 +119,12 @@ marshmallow==2.21.0 # via -r src/requirements.in packaging==21.3 # via + # build # drf-yasg # redis pep517==0.12.0 - # via pip-tools -pip-tools==6.1.0 + # via build +pip-tools==6.9.0 # via -r src/requirements.in portalocker==2.4.0 # via concurrent-log-handler @@ -209,7 +213,9 @@ swagger-spec-validator==2.7.4 titlecase==2.3 # via -r src/requirements.in tomli==2.0.1 - # via pep517 + # via + # build + # pep517 typing-extensions==4.1.1 # via # async-timeout @@ -227,6 +233,8 @@ webcolors==1.11.1 # via jsonschema werkzeug==1.0.1 # via -r src/requirements.in +wheel==0.37.1 + # via pip-tools wrapt==1.14.0 # via deprecated xmltodict==0.12.0 From 432928acaeab08dbe366fdd8e7595b8548742a7e Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Thu, 24 Nov 2022 09:52:10 +0100 Subject: [PATCH 30/34] change: rename PORTFOLIO_AUTOSUGGEST_PRIMOAPIKEY to PRIMO_API_KEY --- src/portfolio/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index 157c6167..92a7e6b0 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -426,8 +426,8 @@ EN_LABELS_TITLE_CASE = env.bool('EN_LABELS_TITLE_CASE', default=True) PRIMO_API_URL = 'https://apigw.obvsg.at/primo/v1/search' -PRIMO_API_KEY = env.str('PORTFOLIO_AUTOSUGGEST_PRIMOAPIKEY', default='') ANGEWANDTE_API_KEY = env.str('ANGEWANDTE_API_KEY', default='') +PRIMO_API_KEY = env.str('PRIMO_API_KEY', default='') GEONAMES_USER = env.str('GEONAMES_USER', default=None) PELIAS_API_KEY = env.str('PELIAS_API_KEY', default=None) PELIAS_API_URL = env.str('PELIAS_API_URL', default='https://api.geocode.earth/v1') From a6cf9f92c1328fe8273bb7fde3230af6cf5a76ab Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Thu, 24 Nov 2022 09:54:08 +0100 Subject: [PATCH 31/34] feat: make PRIMO_API_URL configurable --- src/portfolio/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/portfolio/settings.py b/src/portfolio/settings.py index 92a7e6b0..2233b1ad 100644 --- a/src/portfolio/settings.py +++ b/src/portfolio/settings.py @@ -425,8 +425,8 @@ EN_LABELS_TITLE_CASE = env.bool('EN_LABELS_TITLE_CASE', default=True) -PRIMO_API_URL = 'https://apigw.obvsg.at/primo/v1/search' ANGEWANDTE_API_KEY = env.str('ANGEWANDTE_API_KEY', default='') +PRIMO_API_URL = env.str('PRIMO_API_URL', default='https://apigw.obvsg.at/primo/v1/search') PRIMO_API_KEY = env.str('PRIMO_API_KEY', default='') GEONAMES_USER = env.str('GEONAMES_USER', default=None) PELIAS_API_KEY = env.str('PELIAS_API_KEY', default=None) From df249ae8bbd52208c134d5fa3a57a03722a6cc76 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Thu, 24 Nov 2022 09:55:11 +0100 Subject: [PATCH 32/34] chore: add missing env variables to env-skel --- src/portfolio/env-skel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/portfolio/env-skel b/src/portfolio/env-skel index cd6c6647..8db9ca3a 100644 --- a/src/portfolio/env-skel +++ b/src/portfolio/env-skel @@ -30,7 +30,10 @@ SITE_URL=https://url/ # CORS_ALLOWED_ORIGINS= # OPEN_API_VERSION=2.0 # ACTIVE_SCHEMAS= +# EN_LABELS_TITLE_CASE=True # ANGEWANDTE_API_KEY= +# PRIMO_API_URL= +# PRIMO_API_KEY= # GEONAMES_USER= # PELIAS_API_KEY= # PELIAS_API_URL=https://api.geocode.earth/v1 @@ -38,6 +41,8 @@ SITE_URL=https://url/ # PELIAS_FOCUS_POINT_LAT=48.208126 # PELIAS_FOCUS_POINT_LON=16.382464 # USER_QUOTA=1073741824 +# CLAMAV_ENABLED=True +# CLAMAV_TCP_PORT=3310 # POSTGRES_PORT=5432 # POSTGRES_PASSWORD=password # DOCS_URL=docs/ @@ -49,3 +54,4 @@ SITE_URL=https://url/ # SHOWROOM_REPO_ID= # USER_PREFERENCES_API_BASE= # USER_PREFERENCES_API_KEY= +# SENTRY_TRACES_SAMPLE_RATE=0.2 From b25a6ab3eabd6da2a7cee1e819b726dcb56f2d93 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Thu, 24 Nov 2022 10:01:23 +0100 Subject: [PATCH 33/34] docs: update CHANGELOG --- CHANGELOG.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6959dc21..23031969 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,16 +4,63 @@ ### Added -- Send 301 for retrieve requests in EntryViewSet with old entry ids -- Added `CAS_CHECK_NEXT` environment variable for development +- Added autosuggest route for Primo API +- Added `CAS_CHECK_NEXT` environment variable for development and documentation for it - Added `first_name` and `last_name` to API user response +- Added ClamAV and scan uploaded media objects +- Added bulk creation of entries for importer +- Added project mapping in skosmos.py +- Added `get_preflabel_via_uri` in skosmos.py +- Added management command to update all labels +- Added management commands to evaluate keywords usage of published entries ### Changed -- **BREAKING**: Updated shortuuid to 1.0.1 +- **BREAKING**: Update all labels on every update of Portfolio +- **BREAKING**: Adapted API user response to use `request.user.get_full_name()` instead of the CAS attribute `display_name` to be consistent over multiple Portfolio instances +- Updated pre-commit configuration + +### Fixed + +- Fixed `fix_keywords` management command + +## 1.2.1 + +### Fixed + +- Fixed CC licenses label in API + +## 1.2 + +### Added + +- Send 301 for retrieve requests in EntryViewSet with old entry ids +- Added Showroom connector +- Added connection to User Preferences API +- Added support for Sentry +- Added possibility to sort media objects +- Added possibility to feature a media object +- Added showroom_id field to Entry model +- Added support for nginx crop and resize in media_server +- Added crop and resize to dev config of nginx +- Added management command to push entries to Showroom +- Added management command to manually start the conversion process of a media object +- Added management command to fix missing previews +- Added management commands for fixing migrations issues +- Added documentation of Showroom settings + +### Changed + +- **BREAKING**: Updated shortuuid to 1.0.8 - **BREAKING**: Migrated all existing shortuuids to new format - **BREAKING**: Migrated all existing media directories -- **BREAKING**: Adapted API user response to use `request.user.get_full_name()` to be consistent over Portfolio instances +- Changed search from TrigramSimilarity to TrigramWordSimilarity +- Increased max_length of FileField in Media model +- Adapted dev config of nginx + +### Fixed + +- Fixed prefLabel caching in skosmos.py ## 1.1.6 From 021a6723636d4ada139e3bafde391cca2632ec17 Mon Sep 17 00:00:00 2001 From: Philipp Mayer Date: Thu, 24 Nov 2022 10:06:49 +0100 Subject: [PATCH 34/34] docs: update CHANGELOG for 1.3 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23031969..a6f2acf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## unreleased +## 1.3 ### Added