Merge branch 'release/1.3'

.pre-commit-config.yaml

@@ -2,47 +2,7 @@ default_language_version:
   python: python3
 exclude: (/migrations/|manage.py|docs/source/conf.py)
 repos:
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
+  - repo: https://github.com/base-angewandte/pre-commit-hooks
+    rev: py3.7
     hooks:
-      - id: pyupgrade
-        args: [--py37-plus]
-  - repo: https://github.com/timothycrosley/isort
-    rev: 5.10.1
-    hooks:
-      - id: isort
-  - repo: https://github.com/psf/black
-    rev: 21.12b0
-    hooks:
-      - id: black
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
-    hooks:
-      - id: double-quote-string-fixer
-      - id: end-of-file-fixer
-      - id: check-yaml
-  - repo: https://github.com/myint/docformatter
-    rev: v1.4
-    hooks:
-      - id: docformatter
-        args: [--in-place]
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
-    hooks:
-      - id: flake8
-        additional_dependencies: [flake8-bugbear==22.1.11, pep8-naming==0.12.1]
-  - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.1
-    hooks:
-      - id: bandit
-        exclude: tests.py
-  - repo: https://github.com/IamTheFij/docker-pre-commit
-    rev: v2.0.1
-    hooks:
-      - id: docker-compose-check
-      - id: hadolint
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.5.1
-    hooks:
-      - id: prettier
-        types_or: [markdown, yaml]
+      - id: base-hooks

CHANGELOG.md

@@ -1,16 +1,66 @@
 # Changelog
 
-## unreleased
+## 1.3
+
+### Added
+
+- Added autosuggest route for Primo API
+- Added `CAS_CHECK_NEXT` environment variable for development and documentation for it
+- Added `first_name` and `last_name` to API user response
+- Added ClamAV and scan uploaded media objects
+- Added bulk creation of entries for importer
+- Added project mapping in skosmos.py
+- Added `get_preflabel_via_uri` in skosmos.py
+- Added management command to update all labels
+- Added management commands to evaluate keywords usage of published entries
+
+### Changed
+
+- **BREAKING**: Update all labels on every update of Portfolio
+- **BREAKING**: Adapted API user response to use `request.user.get_full_name()` instead of the CAS attribute `display_name` to be consistent over multiple Portfolio instances
+- Updated pre-commit configuration
+
+### Fixed
+
+- Fixed `fix_keywords` management command
+
+## 1.2.1
+
+### Fixed
+
+- Fixed CC licenses label in API
+
+## 1.2
 
 ### Added
 
 - Send 301 for retrieve requests in EntryViewSet with old entry ids
+- Added Showroom connector
+- Added connection to User Preferences API
+- Added support for Sentry
+- Added possibility to sort media objects
+- Added possibility to feature a media object
+- Added showroom_id field to Entry model
+- Added support for nginx crop and resize in media_server
+- Added crop and resize to dev config of nginx
+- Added management command to push entries to Showroom
+- Added management command to manually start the conversion process of a media object
+- Added management command to fix missing previews
+- Added management commands for fixing migrations issues
+- Added documentation of Showroom settings
 
 ### Changed
 
-- **BREAKING**: Updated shortuuid to 1.0.1
+- **BREAKING**: Updated shortuuid to 1.0.8
 - **BREAKING**: Migrated all existing shortuuids to new format
 - **BREAKING**: Migrated all existing media directories
+- Changed search from TrigramSimilarity to TrigramWordSimilarity
+- Increased max_length of FileField in Media model
+- Adapted dev config of nginx
+
+### Fixed
+
+- Fixed prefLabel caching in skosmos.py
 
 ## 1.1.6
 

Makefile

@@ -40,13 +40,17 @@ restart-gunicorn:
 restart-rq:
 	docker-compose restart portfolio-rq-worker-1 portfolio-rq-worker-2 portfolio-rq-worker-3 portfolio-rq-scheduler
 
-update: git-update init init-rq init-static restart-gunicorn restart-rq build-docs
+update-labels:
+	docker-compose exec portfolio-django python manage.py update_labels
+
+update: git-update init init-rq init-static restart-gunicorn restart-rq build-docs update-labels
 
 start-dev:
 	docker-compose pull --ignore-pull-failures
 	docker-compose up -d \
 		portfolio-redis \
 		portfolio-postgres \
+		portfolio-clamav \
 		portfolio-lool
 
 start-dev-docker:
@@ -55,6 +59,7 @@ start-dev-docker:
 	docker-compose up -d \
 		portfolio-redis \
 		portfolio-postgres \
+		portfolio-clamav \
 		portfolio-lool \
 		portfolio-django
 	docker logs -f portfolio-django

docker-compose.override.dev.yml

@@ -12,6 +12,10 @@ services:
     ports:
       - "127.0.0.1:8200:8200"
 
+  portfolio-clamav:
+    ports:
+      - "127.0.0.1:3310:3310"
+
   portfolio-lool:
     ports:
       - "127.0.0.1:9980:9980"

docker-compose.yml

@@ -104,6 +104,13 @@ services:
     restart: always
     command: python manage.py rqscheduler
 
+  portfolio-clamav:
+    image: clamav/clamav:stable
+    container_name: portfolio-clamav
+    networks:
+      - portfolionet
+    restart: always
+
   portfolio-lool:
     image: collabora/code:4.2.3.1
     container_name: portfolio-lool

docs/source/configuration.md

@@ -78,6 +78,12 @@ Default: `True`
 
 If you are running Portfolio behind nginx, this has to be `True`. For local development, set this to `False`.
 
+### `CAS_CHECK_NEXT`
+
+Default: `True`
+
+If you want to use a non-local `CAS_REDIRECT_URL`, e.g. when redirecting to the frontend in a development setup, set this to `True`.
+
 ### `CAS_REDIRECT_URL`
 
 Default: `FORCE_SCRIPT_NAME` if defined, else `/`
@@ -399,7 +405,7 @@ a local dev environment it might rather be something like
 
 Default: `None`
 
-The API key for this Portfolio instance, as it is defined in the Showroom admin. 
+The API key for this Portfolio instance, as it is defined in the Showroom admin.
 
 #### `SHOWROOM_REPO_ID`
 

src/api/mixins.py

@@ -0,0 +1,25 @@
+from drf_yasg import openapi
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework.decorators import action
+from rest_framework.response import Response
+
+
+class CountModelMixin:
+    """Count a queryset."""
+
+    @swagger_auto_schema(manual_parameters=[], responses={200: openapi.Response('')})
+    @action(detail=False, filter_backends=[], pagination_class=None)
+    def count(self, request, *args, **kwargs):
+        queryset = self.filter_queryset(self.get_queryset())
+        content = {'count': queryset.count()}
+        return Response(content)
+
+
+class CreateListMixin:
+    """Allows bulk creation of a resource."""
+
+    def get_serializer(self, *args, **kwargs):
+        if isinstance(kwargs.get('data', {}), list):
+            kwargs['many'] = True
+
+        return super().get_serializer(*args, **kwargs)

src/api/serializers/entry.py

@@ -3,6 +3,7 @@
 from drf_yasg.utils import swagger_serializer_method
 from rest_framework import serializers
 
+from django.db.models.signals import post_save
 from django.urls import reverse_lazy
 from django.utils.text import format_lazy
 from django.utils.translation import gettext_lazy as _
@@ -44,6 +45,17 @@ class RelationsSerializer(serializers.Serializer):
     to = RelatedEntrySerializer(read_only=True)
 
 
+class EntryBulkCreateSerializer(serializers.ListSerializer):
+    def create(self, validated_data):
+        entries_data = [Entry(**data) for data in validated_data]
+        entries = Entry.objects.bulk_create(entries_data)
+        # send post_save signal for published entries so they are pushed to Showroom
+        for entry in entries:
+            if entry.published:
+                post_save.send(Entry, instance=entry, created=True)
+        return entries
+
+
 class EntrySerializer(CleanModelSerializer, SwaggerMetaModelSerializer):
     owner = serializers.HiddenField(default=serializers.CurrentUserDefault())
     parents = serializers.SerializerMethodField()
@@ -53,6 +65,7 @@ class EntrySerializer(CleanModelSerializer, SwaggerMetaModelSerializer):
 
     class Meta:
         model = Entry
+        list_serializer_class = EntryBulkCreateSerializer
         fields = '__all__'
         swagger_meta_attrs = {
             'id': OrderedDict([('hidden', True)]),

src/api/views.py

@@ -40,6 +40,7 @@
 from media_server.models import get_media_for_entry, update_media_order_for_entry
 from media_server.utils import get_free_space_for_user
 
+from .mixins import CountModelMixin, CreateListMixin
 from .serializers.entry import EntrySerializer
 from .serializers.relation import RelationSerializer
 from .yasg import (
@@ -84,17 +85,6 @@ class StandardLimitOffsetPagination(LimitOffsetPagination):
     # offset_query_param = 'skip'
 
 
-class CountModelMixin:
-    """Count a queryset."""
-
-    @swagger_auto_schema(manual_parameters=[], responses={200: openapi.Response('')})
-    @action(detail=False, filter_backends=[], pagination_class=None)
-    def count(self, request, *args, **kwargs):
-        queryset = self.filter_queryset(self.get_queryset())
-        content = {'count': queryset.count()}
-        return Response(content)
-
-
 class EntryFilter(FilterSet):
     type = CharFilter(field_name='type', lookup_expr='source__iexact')
 
@@ -131,7 +121,7 @@ class Meta:
     ),
     name='list',
 )
-class EntryViewSet(viewsets.ModelViewSet, CountModelMixin):
+class EntryViewSet(CreateListMixin, viewsets.ModelViewSet, CountModelMixin):
     """
     retrieve:
     Returns a certain entry.
@@ -330,8 +320,10 @@ def user_information(request, *args, **kwargs):
     attributes = request.session.get('attributes', {})
     data = {
         'uuid': request.user.username,
-        'name': attributes.get('display_name'),
-        'email': attributes.get('email'),
+        'name': request.user.get_full_name(),
+        'first_name': request.user.first_name,
+        'last_name': request.user.last_name,
+        'email': request.user.email,
         'permissions': attributes.get('permissions') or [],
         'groups': attributes.get('groups') or [],
         'space': get_free_space_for_user(request.user) if request.user else None,

src/core/management/commands/fix_keywords.py

@@ -1,7 +1,6 @@
 from progressbar import progressbar
 from titlecase import titlecase
 
-from django.conf import settings
 from django.core.management.base import BaseCommand
 
 from core.models import Entry
@@ -24,7 +23,7 @@ def handle(self, *args, **options):
                             if 'disciplines' in kw['source']:
                                 project = 'disciplines'
                             else:
-                                project = settings.VOC_ID
+                                project = 'basekw'
                             e.keywords[idx]['label'] = {
                                 'de': get_preflabel(concept, project=project, graph=f'{graph}/', lang='de'),
                                 'en': titlecase(get_preflabel(concept, project=project, graph=f'{graph}/', lang='en')),

src/core/management/commands/keywords_usage.py

@@ -0,0 +1,69 @@
+import csv
+from collections import Counter
+from datetime import date
+
+from progressbar import progressbar
+from titlecase import titlecase
+
+from django.core.management.base import BaseCommand
+
+from core.models import Entry
+from core.skosmos import get_preflabel
+
+
+class Command(BaseCommand):
+    help = 'Return usage of keywords of published entries'
+
+    def handle(self, *args, **options):
+        keywords_list = []
+        for e in Entry.objects.filter(published=True):
+            if e.keywords:
+                for kw in e.keywords:
+                    if kw.get('source'):
+                        keywords_list.append(kw['source'])
+                    else:
+                        keywords_list.append(kw['label']['en'])
+
+        counter = Counter(keywords_list)
+
+        today = date.today().strftime('%d-%m-%Y')
+
+        with open(f'export/{today}_keywords_usage.csv', mode='w') as csvfile:
+            csv_writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
+            csv_writer.writerow(
+                [
+                    'count',
+                    'source',
+                    'label_en',
+                    'label_de',
+                ]
+            )
+            for kw, num in progressbar(counter.most_common()):
+                if kw.startswith('http'):
+                    graph, concept = kw.rsplit('/', 1)
+                    if 'disciplines' in kw:
+                        project = 'disciplines'
+                    else:
+                        project = 'basekw'
+                    label_en = titlecase(get_preflabel(concept, project=project, graph=f'{graph}/', lang='en'))
+                    label_de = get_preflabel(concept, project=project, graph=f'{graph}/', lang='de')
+
+                    csv_writer.writerow(
+                        [
+                            num,
+                            kw,
+                            label_en,
+                            label_de,
+                        ]
+                    )
+                else:
+                    csv_writer.writerow(
+                        [
+                            num,
+                            '',
+                            kw,
+                            kw,
+                        ]
+                    )
+
+        self.stdout.write(self.style.SUCCESS('Successfully exported keywords usage'))