From cc19670058a850df04a52ec00cbc046484418d93 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Mon, 30 Aug 2021 18:00:26 +0530 Subject: [PATCH 01/37] Replace `str` with `uuid` to narrow matches --- openverse-api/catalog/urls/audio.py | 8 ++++---- openverse-api/catalog/urls/images.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/openverse-api/catalog/urls/audio.py b/openverse-api/catalog/urls/audio.py index 09c2aac4b..b103fc2f8 100644 --- a/openverse-api/catalog/urls/audio.py +++ b/openverse-api/catalog/urls/audio.py @@ -16,22 +16,22 @@ name='audio-stats' ), path( - '', + '', AudioDetail.as_view(), name='audio-detail' ), path( - '/thumb', + '/thumb', AudioArt.as_view(), name='audio-thumb' ), path( - '/recommendations', + '/recommendations', RelatedAudio.as_view(), name='audio-related' ), path( - '/waveform', + '/waveform', AudioWaveform.as_view(), name='audio-waveform' ), diff --git a/openverse-api/catalog/urls/images.py b/openverse-api/catalog/urls/images.py index 4595a86ee..a0de32f4a 100644 --- a/openverse-api/catalog/urls/images.py +++ b/openverse-api/catalog/urls/images.py @@ -22,22 +22,22 @@ name='image-oembed' ), path( - '', + '', ImageDetail.as_view(), name='image-detail' ), path( - '/thumb', + '/thumb', ProxiedImage.as_view(), name='image-thumb' ), path( - '/recommendations', + '/recommendations', RelatedImage.as_view(), name='image-related' ), path( - '/report', + '/report', ReportImageView.as_view(), name='report-image' ), From 022ad42e83f23bcd2b7d0c43aa5a9a183d30839e Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Mon, 30 Aug 2021 18:00:49 +0530 Subject: [PATCH 02/37] Define a helper function to raise `APIExceptions` from views --- openverse-api/catalog/api/utils/exceptions.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/openverse-api/catalog/api/utils/exceptions.py b/openverse-api/catalog/api/utils/exceptions.py index 12add0003..ecb39b15c 100644 --- a/openverse-api/catalog/api/utils/exceptions.py +++ b/openverse-api/catalog/api/utils/exceptions.py @@ -1,4 +1,5 @@ from rest_framework import status +from rest_framework.exceptions import APIException from rest_framework.response import Response """ Override the presentation of ValidationErrors, which are deeply nested and @@ -51,3 +52,23 @@ def input_error_response(errors): 'fields': fields } ) + + +def get_api_exception(error_message, response_code=500, error_code=None): + """ + Returns an instance of a subclass of ``APIException`` with the given error + message, error code and response status code. + + The returned object should be used with the ``raise`` keyword. + + :param error_message: the detailed error message shown to the user + :param response_code: the HTTP response status code + :param error_code: the codename of the error + :return: an instance of a subclass of ``APIException`` + """ + + class SubAPIException(APIException): + status_code = response_code + default_detail = error_message + default_code = error_code + return SubAPIException() From 064ccbec508e2274821a11ee1e9c3c66537e03f4 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Mon, 30 Aug 2021 18:01:23 +0530 Subject: [PATCH 03/37] Define a serializer for the audio waveform endpoint --- .../catalog/api/serializers/audio_serializers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index 8c2ffacfc..7130959c8 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -188,6 +188,17 @@ def create(self, validated_data): return AudioReport.objects.create(**validated_data) +class AudioWaveformSerializer(serializers.Serializer): + len = serializers.SerializerMethodField() + points = serializers.ListField( + serializers.FloatField(min_value=0, max_value=1) + ) + + @staticmethod + def get_len(obj): + return len(obj.get('points', [])) + + class AboutAudioSerializer(AboutMediaSerializer): """ Used by `AudioStats`. From 40a4eb499d72387d59921225c33e47a9d6536cf8 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 31 Aug 2021 15:22:50 +0530 Subject: [PATCH 04/37] Add cURL example for the OEmbed endpoint --- openverse-api/catalog/api/examples/__init__.py | 1 + openverse-api/catalog/api/examples/image_requests.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/openverse-api/catalog/api/examples/__init__.py b/openverse-api/catalog/api/examples/__init__.py index 8b2acc008..e602aca1c 100644 --- a/openverse-api/catalog/api/examples/__init__.py +++ b/openverse-api/catalog/api/examples/__init__.py @@ -19,6 +19,7 @@ image_detail_curl, image_stats_curl, report_image_curl, + oembed_list_curl, ) from catalog.api.examples.image_responses import ( image_search_200_example, diff --git a/openverse-api/catalog/api/examples/image_requests.py b/openverse-api/catalog/api/examples/image_requests.py index 2cb651fd0..3a3874b02 100644 --- a/openverse-api/catalog/api/examples/image_requests.py +++ b/openverse-api/catalog/api/examples/image_requests.py @@ -46,3 +46,8 @@ # Report an issue about image ID 7c829a03-fb24-4b57-9b03-65f43ed19395 curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "identifier": "7c829a03-fb24-4b57-9b03-65f43ed19395", "description": "This image contains sensitive content"}' https://api.openverse.engineering/v1/images/7c829a03-fb24-4b57-9b03-65f43ed19395/report """ # noqa + +oembed_list_curl = """ +# Retrieve embedded content from image URL (https://wordpress.org/openverse/photos/7c829a03-fb24-4b57-9b03-65f43ed19395) +curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" https://api.openverse.engineering/v1/oembed/?url=https://wordpress.org/openverse/photos/7c829a03-fb24-4b57-9b03-65f43ed19395 +""" # noqa \ No newline at end of file From c45d395ddcc2a4a778013ea041cfbfcb16d168ce Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 31 Aug 2021 15:25:32 +0530 Subject: [PATCH 05/37] Create serializer for `ContentProvider` --- .../api/serializers/provider_serializers.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 openverse-api/catalog/api/serializers/provider_serializers.py diff --git a/openverse-api/catalog/api/serializers/provider_serializers.py b/openverse-api/catalog/api/serializers/provider_serializers.py new file mode 100644 index 000000000..f5da8b502 --- /dev/null +++ b/openverse-api/catalog/api/serializers/provider_serializers.py @@ -0,0 +1,35 @@ +from rest_framework import serializers + +from catalog.api.models import ContentProvider, SourceLogo + + +class ProviderSerializer(serializers.ModelSerializer): + source_name = serializers.CharField(source='provider_identifier') + display_name = serializers.CharField(source='provider_name') + source_url = serializers.URLField(source='domain_name') + logo_url = serializers.SerializerMethodField() + media_count = serializers.SerializerMethodField() + + class Meta: + model = ContentProvider + fields = [ + 'source_name', + 'display_name', + 'source_url', + 'logo_url', + 'media_count', + ] + + def get_logo_url(self, obj): + try: + source_logo = obj.sourcelogo + except SourceLogo.DoesNotExist: + return None + logo_path = source_logo.image.url + request = self.context.get('request') + if request is not None: + return request.build_absolute_uri(logo_path) + + def get_media_count(self, obj): + source_counts = self.context.get('source_counts') + return source_counts.get(obj.provider_identifier) From a2f700e2da2666a86630dfd47561688bf6326fc1 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 31 Aug 2021 15:26:09 +0530 Subject: [PATCH 06/37] Move pagination attributes from search query serializer to paginator class --- .../api/serializers/media_serializers.py | 24 --------- openverse-api/catalog/api/utils/pagination.py | 54 +++++++++++++++++++ 2 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 openverse-api/catalog/api/utils/pagination.py diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index 442e50bd7..db6d09ad1 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -97,8 +97,6 @@ class MediaSearchQueryStringSerializer(serializers.Serializer): 'q', 'license', 'license_type', - 'page', - 'page_size', 'creator', 'tags', 'title', @@ -132,17 +130,6 @@ class MediaSearchQueryStringSerializer(serializers.Serializer): f"`{list(license_helpers.LICENSE_GROUPS.keys())}`", required=False, ) - page = serializers.IntegerField( - label="page number", - help_text="The page number to retrieve.", - default=1 - ) - page_size = serializers.IntegerField( - label="page size", - help_text="The number of results to return in the requested page. " - "Should be an integer between 1 and 500.", - default=20 - ) creator = serializers.CharField( label="creator", help_text="Search by creator only. Cannot be used with `q`.", @@ -205,17 +192,6 @@ def validate_license_type(value): """ return _validate_lt(value) - @staticmethod - def validate_page(value): - return _validate_page(value) - - @staticmethod - def validate_page_size(value): - if 1 <= value <= 500: - return value - else: - return 20 - def validate_creator(self, value): return self.validate_q(value) diff --git a/openverse-api/catalog/api/utils/pagination.py b/openverse-api/catalog/api/utils/pagination.py new file mode 100644 index 000000000..b9064e129 --- /dev/null +++ b/openverse-api/catalog/api/utils/pagination.py @@ -0,0 +1,54 @@ +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination + +from catalog.api.utils.exceptions import get_api_exception + + +class StandardPagination(PageNumberPagination): + page_size_query_param = 'page_size' + page_query_param = 'page' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.result_count = None # populated later + self.page_count = None # populated later + + self._page_size = 20 + self._page = None + + @property + def page_size(self): + """the number of results to show in one page""" + return self._page_size + + @page_size.setter + def page_size(self, value): + if value is None: + return + value = int(value) # convert str params to int + if value <= 0 or value > 500: + raise get_api_exception(f'Page size must be between 0 & 500.', 400) + self._page_size = value + + @property + def page(self): + """the current page number being served""" + return self._page + + @page.setter + def page(self, value): + if value is None: + value = 1 + value = int(value) # convert str params to int + if value <= 0: + raise get_api_exception('Page must be greater than 0.', 400) + self._page = value + + def get_paginated_response(self, data): + return Response({ + 'result_count': self.result_count, + 'page_count': self.page_count, + 'page_size': self.page_size, + 'page': self.page, + 'results': data, + }) From 0e03571a230208f89c7a1f3dd4cd926b1e98832e Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 31 Aug 2021 15:27:24 +0530 Subject: [PATCH 07/37] Cleanup OEmbed input and output serializers --- .../api/serializers/image_serializers.py | 65 +++++++++++++------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index a81ea8529..3fe7b5c49 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -2,7 +2,7 @@ from rest_framework import serializers from catalog.api.controllers.search_controller import get_sources -from catalog.api.models import ImageReport +from catalog.api.models import Image, ImageReport from catalog.api.serializers.media_serializers import ( _add_protocol, _validate_enum, @@ -142,33 +142,68 @@ class ImageSearchResultsSerializer(MediaSearchResultsSerializer): ) -class OembedResponseSerializer(serializers.Serializer): +class OembedRequestSerializer(serializers.Serializer): + """ Parse and validate Oembed parameters. """ + url = serializers.CharField( + help_text="The link to an image.", + required=True, + ) + + @staticmethod + def validate_url(value): + return _add_protocol(value) + + +class OembedSerializer(serializers.ModelSerializer): """ The embedded content from a specified image URL. """ - version = serializers.IntegerField( - help_text="The image version." + version = serializers.ReadOnlyField( + help_text="The image version.", + default='1.0', ) - type = serializers.CharField( - help_text="Type of data." + type = serializers.ReadOnlyField( + help_text="Type of data.", + default='photo', ) - width = serializers.IntegerField( + width = serializers.SerializerMethodField( help_text="The width of the image in pixels." ) - height = serializers.IntegerField( + height = serializers.SerializerMethodField( help_text="The height of the image in pixels." ) title = serializers.CharField( help_text="The name of image." ) author_name = serializers.CharField( - help_text="The name of author for image." + help_text="The name of author for image.", + source='creator', ) author_url = serializers.URLField( - help_text="A direct link to the author." + help_text="A direct link to the author.", + source='creator_url', ) license_url = serializers.URLField( help_text="A direct link to the license for image." ) + class Meta: + model = Image + fields = [ + 'version', + 'type', + 'width', + 'height', + 'title', + 'author_name', + 'author_url', + 'license_url', + ] + + def get_width(self, obj): + return self.context.get('width', obj.width) + + def get_height(self, obj): + return self.context.get('height', obj.height) + class WatermarkQueryStringSerializer(serializers.Serializer): embed_metadata = serializers.BooleanField( @@ -197,16 +232,6 @@ def create(self, validated_data): return ImageReport.objects.create(**validated_data) -class OembedSerializer(serializers.Serializer): - """ Parse and validate Oembed parameters. """ - url = serializers.URLField( - help_text="The link to an image." - ) - - def validate_url(self, value): - return _add_protocol(value) - - class AboutImageSerializer(AboutMediaSerializer): """ Used by `ImageStats`. From 870908a59f6456c028280e43549f899931622d72 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 31 Aug 2021 15:29:15 +0530 Subject: [PATCH 08/37] Create parent viewset for all media types --- .../catalog/api/views/media_views.py | 254 +++++++++--------- 1 file changed, 128 insertions(+), 126 deletions(-) diff --git a/openverse-api/catalog/api/views/media_views.py b/openverse-api/catalog/api/views/media_views.py index 831095c54..e100e1b7c 100644 --- a/openverse-api/catalog/api/views/media_views.py +++ b/openverse-api/catalog/api/views/media_views.py @@ -2,36 +2,22 @@ from django.conf import settings from django.http.response import HttpResponse -from rest_framework.authentication import BasicAuthentication -from rest_framework.generics import GenericAPIView -from rest_framework.mixins import RetrieveModelMixin -from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.views import APIView +from rest_framework.viewsets import ReadOnlyModelViewSet from urllib.error import HTTPError from urllib.request import urlopen from catalog.api.controllers import search_controller from catalog.api.controllers.search_controller import get_sources -from catalog.api.models import ContentProvider, SourceLogo -from catalog.api.utils.exceptions import input_error_response -from catalog.api.utils.throttle import OneThousandPerMinute +from catalog.api.serializers.provider_serializers import ProviderSerializer +from catalog.api.utils.pagination import StandardPagination +from catalog.api.models import ContentProvider +from catalog.api.utils.exceptions import get_api_exception from catalog.custom_auto_schema import CustomAutoSchema log = logging.getLogger(__name__) -FOREIGN_LANDING_URL = 'foreign_landing_url' -CREATOR_URL = 'creator_url' -RESULTS = 'results' -PAGE = 'page' -PAGESIZE = 'page_size' -FILTER_DEAD = 'filter_dead' -QA = 'qa' -SUGGESTIONS = 'suggestions' -RESULT_COUNT = 'result_count' -PAGE_COUNT = 'page_count' -PAGE_SIZE = 'page_size' - refer_sample = """ You can refer to the cURL request samples for examples on how to consume this endpoint. @@ -68,10 +54,12 @@ def _get_user_ip(request): return ip -class SearchMedia(APIView): - swagger_schema = CustomAutoSchema - search_description = ( +class MediaSearch: + desc = ( """ +Results are ranked in order of relevance and paginated on the basis of the +`page` param. The `page_size` param controls the total number of pages. + Although there may be millions of relevant records, only the most relevant several thousand records can be viewed. This is by design: the search endpoint should be used to find the top 10,000 most relevant @@ -88,21 +76,83 @@ class SearchMedia(APIView): f'{refer_sample}' ) - def _get(self, - request, - default_index, qa_index, - query_serializer, media_serializer, result_serializer): - params = query_serializer(data=request.query_params) + +class MediaStats: + desc = ( + """ +You can use this endpoint to get details about content providers such as +`source_name`, `display_name`, and `source_url` along with a count of the number +of individual items indexed from them. +""" # noqa + f'{refer_sample}' + ) + + +class MediaDetail: + desc = refer_sample + + +class MediaRelated: + desc = refer_sample + + +class MediaComplain: + desc = ( + """ +By using this endpoint, you can report a file if it infringes copyright, +contains mature or sensitive content and others. +""" # noqa + f'{refer_sample}' + ) + + +class MediaViewSet(ReadOnlyModelViewSet): + swagger_schema = CustomAutoSchema + + lookup_field = 'identifier' + # TODO: https://github.com/encode/django-rest-framework/pull/6789 + lookup_value_regex = r'[0-9a-f\-]{36}' # highly simplified approximation + + pagination_class = StandardPagination + + # Populate these in the corresponding subclass + model_class = None + query_serializer_class = None + default_index = None + qa_index = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + required_fields = [ + self.model_class, + self.query_serializer_class, + self.default_index, + self.qa_index, + ] + if any(val is None for val in required_fields): + msg = 'Viewset fields are not completely populated.' + raise ValueError(msg) + + def get_queryset(self): + return self.model_class.objects.all() + + # Standard actions + + def list(self, request, *_, **__): + self.paginator.page_size = request.query_params.get('page_size') + page_size = self.paginator.page_size + self.paginator.page = request.query_params.get('page') + page = self.paginator.page + + params = self.query_serializer_class(data=request.query_params) if not params.is_valid(): - return input_error_response(params.errors) + raise get_api_exception('Input is invalid.', 400) hashed_ip = hash(_get_user_ip(request)) - page_param = params.data[PAGE] - page_size = params.data[PAGESIZE] - qa = params.data[QA] - filter_dead = params.data[FILTER_DEAD] + qa = params.validated_data['qa'] + filter_dead = params.validated_data['filter_dead'] - search_index = qa_index if qa else default_index + search_index = self.qa_index if qa else self.default_index try: results, num_pages, num_results = search_controller.search( params, @@ -111,114 +161,66 @@ def _get(self, hashed_ip, request, filter_dead, - page=page_param + page, ) - except ValueError as value_error: - return input_error_response(value_error) - - context = {'request': request} - serialized_results = media_serializer( - results, - many=True, - context=context - ).data - - if len(results) < page_size and num_pages == 0: - num_results = len(results) - response_data = { - RESULT_COUNT: num_results, - PAGE_COUNT: num_pages, - PAGE_SIZE: len(results), - RESULTS: serialized_results + self.paginator.page_count = num_pages + self.paginator.result_count = num_results + except ValueError as e: + raise get_api_exception(getattr(e, 'message', str(e))) + + serializer = self.get_serializer(results, many=True) + return self.get_paginated_response(serializer.data) + + # Extra actions + + @action(detail=False, + serializer_class=ProviderSerializer) + def stats(self, *_, **__): + source_counts = get_sources(self.default_index) + context = self.get_serializer_context() | { + 'source_counts': source_counts, } - serialized_response = result_serializer(data=response_data) - return Response(status=200, data=serialized_response.initial_data) - - -class RelatedMedia(APIView): - swagger_schema = CustomAutoSchema - recommendations_read_description = refer_sample - -class MediaDetail(GenericAPIView, RetrieveModelMixin): - swagger_schema = CustomAutoSchema - lookup_field = 'identifier' - authentication_classes = [BasicAuthentication] - permission_classes = [IsAuthenticatedOrReadOnly] - detail_description = refer_sample - - -class MediaStats(APIView): - swagger_schema = CustomAutoSchema - media_stats_description = ( - """ -You can use this endpoint to get details about content providers such as -`source_name`, `display_name`, and `source_url` along with a count of the number -of individual items indexed from them. -""" # noqa - f'{refer_sample}' - ) + providers = ContentProvider \ + .objects \ + .filter(media_type=self.default_index, filter_content=False) + serializer = self.get_serializer(providers, many=True, context=context) + return Response(serializer.data) - def _get(self, request, index_name): - CODENAME = 'provider_identifier' - NAME = 'provider_name' - FILTER = 'filter_content' - URL = 'domain_name' - ID = 'id' + @action(detail=True) + def related(self, request, identifier=None, *_, **__): + try: + results, num_results = search_controller.related_media( + uuid=identifier, + index=self.default_index, + request=request, + filter_dead=True + ) + self.paginator.result_count = num_results + self.paginator.page_count = 1 + self.paginator.page_size = num_results + except ValueError as e: + raise get_api_exception(getattr(e, 'message', str(e))) - source_data = ContentProvider \ - .objects \ - .filter(media_type=index_name) \ - .values(ID, CODENAME, NAME, FILTER, URL) - source_counts = get_sources(index_name) - - response = [] - for source in source_data: - source_codename = source[CODENAME] - _id = source[ID] - display_name = source[NAME] - filtered = source[FILTER] - source_url = source[URL] - count = source_counts.get(source_codename, None) - try: - source_logo = SourceLogo.objects.get(source_id=_id) - logo_path = source_logo.image.url - full_logo_url = request.build_absolute_uri(logo_path) - except SourceLogo.DoesNotExist: - full_logo_url = None - if not filtered and source_codename in source_counts: - response.append( - { - 'source_name': source_codename, - f'{index_name}_count': count, - 'display_name': display_name, - 'source_url': source_url, - 'logo_url': full_logo_url - } - ) - return Response(status=200, data=response) - - -class ImageProxy(APIView): - swagger_schema = None + serializer = self.get_serializer(results, many=True) + return self.get_paginated_response(serializer.data) - lookup_field = 'identifier' - throttle_classes = [OneThousandPerMinute] + # Helper functions - def _get(self, media_url, width=settings.THUMBNAIL_WIDTH_PX): + @staticmethod + def _get_proxied_image(image_url, width=settings.THUMBNAIL_WIDTH_PX): if width is None: # full size - proxy_upstream = f'{settings.THUMBNAIL_PROXY_URL}/{media_url}' + proxy_upstream = f'{settings.THUMBNAIL_PROXY_URL}/{image_url}' else: proxy_upstream = f'{settings.THUMBNAIL_PROXY_URL}/' \ f'{settings.THUMBNAIL_WIDTH_PX},fit/' \ - f'{media_url}' + f'{image_url}' try: upstream_response = urlopen(proxy_upstream) status = upstream_response.status content_type = upstream_response.headers.get('Content-Type') except HTTPError: - log.info('Failed to render thumbnail: ', exc_info=True) - return HttpResponse(status=500) + raise get_api_exception('Failed to render thumbnail.') response = HttpResponse( upstream_response.read(), From 3c9d2b660568d7ba32c50fa0af8b7ff8949b952f Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:38:01 +0530 Subject: [PATCH 09/37] Add examples for reporting audio files --- openverse-api/catalog/api/examples/__init__.py | 2 ++ openverse-api/catalog/api/examples/audio_requests.py | 6 ++++++ openverse-api/catalog/api/examples/audio_responses.py | 9 +++++++++ openverse-api/catalog/api/examples/image_requests.py | 2 +- openverse-api/catalog/api/examples/image_responses.py | 3 ++- 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/openverse-api/catalog/api/examples/__init__.py b/openverse-api/catalog/api/examples/__init__.py index e602aca1c..e2117ee5d 100644 --- a/openverse-api/catalog/api/examples/__init__.py +++ b/openverse-api/catalog/api/examples/__init__.py @@ -3,6 +3,7 @@ recommendations_audio_read_curl, audio_detail_curl, audio_stats_curl, + report_audio_curl, ) from catalog.api.examples.audio_responses import ( audio_search_200_example, @@ -11,6 +12,7 @@ audio_detail_404_example, recommendations_audio_read_200_example, recommendations_audio_read_404_example, + audio_report_create_201_example, audio_stats_200_example, ) from catalog.api.examples.image_requests import ( diff --git a/openverse-api/catalog/api/examples/audio_requests.py b/openverse-api/catalog/api/examples/audio_requests.py index d632879e2..b62805987 100644 --- a/openverse-api/catalog/api/examples/audio_requests.py +++ b/openverse-api/catalog/api/examples/audio_requests.py @@ -41,3 +41,9 @@ # Get the statistics for audio sources curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/audio/stats """ # noqa + +report_audio_curl = """ +# Report an issue about audio ID 7c829a03-fb24-4b57-9b03-65f43ed19395 +curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "description": "This audio contains sensitive content"}' https://api.openverse.engineering/v1/audio/7c829a03-fb24-4b57-9b03-65f43ed19395/report +""" # noqa + diff --git a/openverse-api/catalog/api/examples/audio_responses.py b/openverse-api/catalog/api/examples/audio_responses.py index 96ea7e061..1bb686ad4 100644 --- a/openverse-api/catalog/api/examples/audio_responses.py +++ b/openverse-api/catalog/api/examples/audio_responses.py @@ -91,6 +91,15 @@ } } +audio_report_create_201_example = { + "application/json": { + "id": 10, + "identifier": "7c829a03-fb24-4b57-9b03-65f43ed19395", + "reason": "mature", + "description": "This audio contains sensitive content" + } +} + audio_stats_200_example = { "application/json": [ { diff --git a/openverse-api/catalog/api/examples/image_requests.py b/openverse-api/catalog/api/examples/image_requests.py index 3a3874b02..b885589a0 100644 --- a/openverse-api/catalog/api/examples/image_requests.py +++ b/openverse-api/catalog/api/examples/image_requests.py @@ -44,7 +44,7 @@ report_image_curl = """ # Report an issue about image ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "identifier": "7c829a03-fb24-4b57-9b03-65f43ed19395", "description": "This image contains sensitive content"}' https://api.openverse.engineering/v1/images/7c829a03-fb24-4b57-9b03-65f43ed19395/report +curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "description": "This image contains sensitive content"}' https://api.openverse.engineering/v1/images/7c829a03-fb24-4b57-9b03-65f43ed19395/report """ # noqa oembed_list_curl = """ diff --git a/openverse-api/catalog/api/examples/image_responses.py b/openverse-api/catalog/api/examples/image_responses.py index ac58ac974..ce4edcc19 100644 --- a/openverse-api/catalog/api/examples/image_responses.py +++ b/openverse-api/catalog/api/examples/image_responses.py @@ -134,8 +134,9 @@ images_report_create_201_example = { "application/json": { - "reason": "mature", + "id": 10, "identifier": "7c829a03-fb24-4b57-9b03-65f43ed19395", + "reason": "mature", "description": "This image contains sensitive content" } } From e0c0ec33ff5ee0410a0c929f631e767c4766470b Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:40:17 +0530 Subject: [PATCH 10/37] Replace `AboutMediaSerializer` superseded by `ProviderSerializer` --- .../api/serializers/audio_serializers.py | 11 --------- .../api/serializers/image_serializers.py | 11 --------- .../api/serializers/media_serializers.py | 16 ------------- .../api/serializers/provider_serializers.py | 23 +++++++++++++++---- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index 7130959c8..74ee3b79b 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -8,7 +8,6 @@ MediaSearchQueryStringSerializer, MediaSearchResultsSerializer, MediaSerializer, - AboutMediaSerializer, ) @@ -197,13 +196,3 @@ class AudioWaveformSerializer(serializers.Serializer): @staticmethod def get_len(obj): return len(obj.get('points', [])) - - -class AboutAudioSerializer(AboutMediaSerializer): - """ - Used by `AudioStats`. - """ - - audio_count = serializers.IntegerField( - help_text="The number of audio files." - ) diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index 3fe7b5c49..8a8a0badd 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -9,7 +9,6 @@ MediaSearchQueryStringSerializer, MediaSearchResultsSerializer, MediaSerializer, - AboutMediaSerializer, ) @@ -230,13 +229,3 @@ def create(self, validated_data): "Description must be at least be 20 characters long" ) return ImageReport.objects.create(**validated_data) - - -class AboutImageSerializer(AboutMediaSerializer): - """ - Used by `ImageStats`. - """ - - image_count = serializers.IntegerField( - help_text="The number of images." - ) diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index db6d09ad1..ede15f152 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -358,22 +358,6 @@ class MediaSearchResultsSerializer(serializers.Serializer): ) -class AboutMediaSerializer(serializers.Serializer): - """ - This serializer represents the response of the media statistics endpoints. - """ - - source_name = serializers.CharField( - help_text="The source of the media." - ) - display_name = serializers.CharField( - help_text="The name of content provider." - ) - source_url = serializers.CharField( - help_text="The actual URL to the `source_name`." - ) - - class ProxiedImageSerializer(serializers.Serializer): """ We want to show 3rd party content securely and under our own native URLs, so diff --git a/openverse-api/catalog/api/serializers/provider_serializers.py b/openverse-api/catalog/api/serializers/provider_serializers.py index f5da8b502..a996dbfba 100644 --- a/openverse-api/catalog/api/serializers/provider_serializers.py +++ b/openverse-api/catalog/api/serializers/provider_serializers.py @@ -4,11 +4,24 @@ class ProviderSerializer(serializers.ModelSerializer): - source_name = serializers.CharField(source='provider_identifier') - display_name = serializers.CharField(source='provider_name') - source_url = serializers.URLField(source='domain_name') - logo_url = serializers.SerializerMethodField() - media_count = serializers.SerializerMethodField() + source_name = serializers.CharField( + source='provider_identifier', + help_text='The source of the media.', + ) + display_name = serializers.CharField( + source='provider_name', + help_text='The name of content provider.', + ) + source_url = serializers.URLField( + source='domain_name', + help_text='The URL of the source.', + ) + logo_url = serializers.SerializerMethodField( + help_text='The URL to a logo of the source.', + ) + media_count = serializers.SerializerMethodField( + help_text='The number of media items indexed from the source.', + ) class Meta: model = ContentProvider From 8d8d2261d2d2c0e86ea59169e70f7343745ec3d9 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:40:56 +0530 Subject: [PATCH 11/37] Add attribute 'page' to search results --- openverse-api/catalog/api/serializers/media_serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index ede15f152..4f6939636 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -356,6 +356,9 @@ class MediaSearchResultsSerializer(serializers.Serializer): page_size = serializers.IntegerField( help_text="The number of items per page." ) + page = serializers.IntegerField( + help_text="The current page number returned in the response." + ) class ProxiedImageSerializer(serializers.Serializer): From 6ec373b76cd6a205da0df83e521577bcb7d49425 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:41:57 +0530 Subject: [PATCH 12/37] Infer reported content identifier from the endpoint URL --- .../catalog/api/serializers/audio_serializers.py | 9 +++++++-- .../catalog/api/serializers/image_serializers.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index 74ee3b79b..cbde630d2 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -161,7 +161,11 @@ def get_waveform(self, obj): class AudioSearchResultsSerializer(MediaSearchResultsSerializer): - """ The full audio search response. """ + """ + The full audio search response. + This serializer is purely representational and not actually used to + serialize the response. + """ results = AudioSerializer( many=True, help_text="An array of audios and their details such as `title`, `id`, " @@ -175,7 +179,8 @@ class AudioSearchResultsSerializer(MediaSearchResultsSerializer): class ReportAudioSerializer(serializers.ModelSerializer): class Meta: model = AudioReport - fields = ('reason', 'identifier', 'description') + fields = ('id', 'identifier', 'reason', 'description') + read_only_fields = ('id', 'identifier',) def create(self, validated_data): if validated_data['reason'] == "other" and \ diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index 8a8a0badd..929ce21da 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -130,7 +130,11 @@ def get_thumbnail(self, obj): class ImageSearchResultsSerializer(MediaSearchResultsSerializer): - """ The full image search response. """ + """ + The full image search response. + This serializer is purely representational and not actually used to + serialize the response. + """ results = ImageSerializer( many=True, help_text="An array of images and their details such as `title`, `id`, " @@ -219,7 +223,8 @@ class WatermarkQueryStringSerializer(serializers.Serializer): class ReportImageSerializer(serializers.ModelSerializer): class Meta: model = ImageReport - fields = ('reason', 'identifier', 'description') + fields = ('id', 'identifier', 'reason', 'description') + read_only_fields = ('id', 'identifier',) def create(self, validated_data): if validated_data['reason'] == "other" and \ From cc009f3a6c9380b0a165a04ffd059f78fe0d39a2 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:57:08 +0530 Subject: [PATCH 13/37] Migrate endpoint documentation to a docs directory --- openverse-api/catalog/api/docs/__init__.py | 0 openverse-api/catalog/api/docs/audio_docs.py | 214 +++++++++++++++ openverse-api/catalog/api/docs/image_docs.py | 260 +++++++++++++++++++ openverse-api/catalog/api/docs/media_docs.py | 70 +++++ 4 files changed, 544 insertions(+) create mode 100644 openverse-api/catalog/api/docs/__init__.py create mode 100644 openverse-api/catalog/api/docs/audio_docs.py create mode 100644 openverse-api/catalog/api/docs/image_docs.py create mode 100644 openverse-api/catalog/api/docs/media_docs.py diff --git a/openverse-api/catalog/api/docs/__init__.py b/openverse-api/catalog/api/docs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openverse-api/catalog/api/docs/audio_docs.py b/openverse-api/catalog/api/docs/audio_docs.py new file mode 100644 index 000000000..fd06ec76e --- /dev/null +++ b/openverse-api/catalog/api/docs/audio_docs.py @@ -0,0 +1,214 @@ +from drf_yasg import openapi + +from catalog.api.docs.media_docs import ( + fields_to_md, + MediaSearch, + MediaStats, + MediaDetail, + MediaRelated, + MediaComplain, +) +from catalog.api.examples import ( + audio_search_curl, + audio_search_200_example, + audio_search_400_example, + recommendations_audio_read_curl, + recommendations_audio_read_200_example, + recommendations_audio_read_404_example, + audio_detail_curl, + audio_detail_200_example, + audio_detail_404_example, + audio_stats_curl, + audio_stats_200_example, + report_audio_curl, + audio_report_create_201_example, +) +from catalog.api.serializers.audio_serializers import ( + AudioSearchQueryStringSerializer, + AudioSearchResultsSerializer, + AudioSerializer, + ReportAudioSerializer, +) +from catalog.api.serializers.error_serializers import ( + InputErrorSerializer, + NotFoundErrorSerializer, +) +from catalog.api.serializers.provider_serializers import ProviderSerializer + + +class AudioSearch(MediaSearch): + desc = f""" +audio_search is an API endpoint to search audio files using a query string. + +By using this endpoint, you can obtain search results based on specified +query and optionally filter results by +{fields_to_md(AudioSearchQueryStringSerializer.fields_names)}. + +{MediaSearch.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=audio_search_200_example, + schema=AudioSearchResultsSerializer(many=True) + ), + "400": openapi.Response( + description="Bad Request", + examples=audio_search_400_example, + schema=InputErrorSerializer + ), + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': audio_search_curl, + }, + ] + + swagger_setup = { + 'operation_id': 'audio_search', + 'operation_description': desc, + 'query_serializer': AudioSearchQueryStringSerializer, + 'responses': responses, + 'code_examples': code_examples + } + + +class AudioStats(MediaStats): + desc = f""" +audio_stats is an API endpoint to get a list of all content providers and their +respective number of audio files in the Openverse catalog. + +{MediaStats.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=audio_stats_200_example, + schema=ProviderSerializer(many=True) + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': audio_stats_curl, + }, + ] + + swagger_setup = { + 'operation_id': 'audio_stats', + 'operation_description': desc, + 'responses': responses, + 'code_examples': code_examples, + } + + +class AudioDetail(MediaDetail): + desc = f""" +audio_detail is an API endpoint to get the details of a specified audio ID. + +By using this endpoint, you can get audio details such as +{fields_to_md(AudioSerializer.fields_names)}. + +{MediaDetail.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=audio_detail_200_example, + schema=AudioSerializer + ), + "404": openapi.Response( + description="OK", + examples=audio_detail_404_example, + schema=NotFoundErrorSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': audio_detail_curl, + }, + ] + + swagger_setup = { + 'operation_id': 'audio_detail', + 'operation_description': desc, + 'responses': responses, + 'code_examples': code_examples, + } + + +class AudioRelated(MediaRelated): + desc = f""" +recommendations_audio_read is an API endpoint to get related audio files +for a specified audio ID. + +By using this endpoint, you can get the details of related audio such as +{fields_to_md(AudioSerializer.fields_names)}. + +{MediaRelated.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=recommendations_audio_read_200_example, + schema=AudioSerializer + ), + "404": openapi.Response( + description="Not Found", + examples=recommendations_audio_read_404_example, + schema=NotFoundErrorSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': recommendations_audio_read_curl, + }, + ] + + swagger_setup = { + 'operation_id': 'audio_related', + 'operation_description': desc, + 'responses': responses, + 'code_examples': code_examples + } + + +class AudioComplain(MediaComplain): + desc = f""" +audio_report_create is an API endpoint to report an issue about a specified +audio ID to Openverse. + +By using this endpoint, you can report an audio file if it infringes copyright, +contains mature or sensitive content and others. + +{MediaComplain.desc}""" # noqa + + responses = { + "201": openapi.Response( + description="OK", + examples=audio_report_create_201_example, + schema=ReportAudioSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': report_audio_curl, + } + ] + + swagger_setup = { + 'operation_id': 'audio_report', + 'operation_description': desc, + 'query_serializer': ReportAudioSerializer, + 'responses': responses, + 'code_examples': code_examples, + } diff --git a/openverse-api/catalog/api/docs/image_docs.py b/openverse-api/catalog/api/docs/image_docs.py new file mode 100644 index 000000000..12903c4b3 --- /dev/null +++ b/openverse-api/catalog/api/docs/image_docs.py @@ -0,0 +1,260 @@ +from drf_yasg import openapi + +from catalog.api.docs.media_docs import ( + fields_to_md, + refer_sample, + MediaSearch, + MediaStats, + MediaDetail, + MediaRelated, + MediaComplain, +) +from catalog.api.examples import ( + image_search_curl, + image_search_200_example, + image_search_400_example, + recommendations_images_read_curl, + recommendations_images_read_200_example, + recommendations_images_read_404_example, + image_detail_curl, + image_detail_200_example, + image_detail_404_example, + image_stats_curl, + image_stats_200_example, + report_image_curl, + images_report_create_201_example, + oembed_list_curl, + oembed_list_200_example, + oembed_list_404_example, +) +from catalog.api.serializers.error_serializers import ( + InputErrorSerializer, + NotFoundErrorSerializer, +) +from catalog.api.serializers.image_serializers import ( + ImageSearchQueryStringSerializer, + ImageSearchResultsSerializer, + ImageSerializer, + ReportImageSerializer, + OembedRequestSerializer, + OembedSerializer, +) +from catalog.api.serializers.provider_serializers import ProviderSerializer + + +class ImageSearch(MediaSearch): + desc = f""" +image_search is an API endpoint to search images using a query string. + +By using this endpoint, you can obtain search results based on specified query +and optionally filter results by +{fields_to_md(ImageSearchQueryStringSerializer.fields_names)}. + +{MediaSearch.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=image_search_200_example, + schema=ImageSearchResultsSerializer + ), + "400": openapi.Response( + description="Bad Request", + examples=image_search_400_example, + schema=InputErrorSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': image_search_curl, + }, + ] + + swagger_setup = { + 'operation_id': 'image_search', + 'operation_description': desc, + 'query_serializer': ImageSearchQueryStringSerializer, + 'responses': responses, + 'code_examples': code_examples, + } + + +class ImageStats(MediaStats): + desc = f""" +image_stats is an API endpoint to get a list of all content providers and their +respective number of images in the Openverse catalog. + +{MediaStats.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=image_stats_200_example, + schema=ProviderSerializer(many=True) + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': image_stats_curl, + } + ] + + swagger_setup = { + 'operation_id': 'image_stats', + 'operation_description': desc, + 'responses': responses, + 'code_examples': code_examples + } + + +class ImageDetail(MediaDetail): + desc = f""" +image_detail is an API endpoint to get the details of a specified image ID. + +By using this endpoint, you can image details such as +{fields_to_md(ImageSerializer.fields_names)}. + +{MediaDetail.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=image_detail_200_example, + schema=ImageSerializer + ), + "404": openapi.Response( + description='Not Found', + examples=image_detail_404_example, + schema=NotFoundErrorSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': image_detail_curl + } + ] + + swagger_setup = { + 'operation_id': "image_detail", + 'operation_description': desc, + 'responses': responses, + 'code_examples': code_examples, + } + + +class ImageRelated(MediaRelated): + desc = f""" +recommendations_images_read is an API endpoint to get related images +for a specified image ID. + +By using this endpoint, you can get the details of related images such as +{fields_to_md(ImageSerializer.fields_names)}. + +{MediaRelated.desc}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=recommendations_images_read_200_example, + schema=ImageSerializer + ), + "404": openapi.Response( + description="Not Found", + examples=recommendations_images_read_404_example, + schema=NotFoundErrorSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': recommendations_images_read_curl + } + ] + + swagger_setup = { + 'operation_id': "image_related", + 'operation_description': desc, + 'responses': responses, + 'code_examples': code_examples + } + + +class ImageComplain(MediaComplain): + desc = f""" +images_report_create is an API endpoint to report an issue about a specified +image ID to Openverse. + +By using this endpoint, you can report an image if it infringes copyright, +contains mature or sensitive content and others. + +{MediaComplain.desc}""" # noqa + + responses = { + "201": openapi.Response( + description="OK", + examples=images_report_create_201_example, + schema=ReportImageSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': report_image_curl, + } + ] + + swagger_setup = { + 'operation_id': 'image_report', + 'operation_description': desc, + 'query_serializer': ReportImageSerializer, + 'responses': responses, + 'code_examples': code_examples, + } + + +class ImageOembed: + desc = f""" +oembed_list is an API endpoint to retrieve embedded content from a +specified image URL. + +By using this endpoint, you can retrieve embedded content such as `version`, +`type`, `width`, `height`, `title`, `author_name`, `author_url` and +`license_url`. + +{refer_sample}""" # noqa + + responses = { + "200": openapi.Response( + description="OK", + examples=oembed_list_200_example, + schema=OembedSerializer + ), + "404": openapi.Response( + description="Not Found", + examples=oembed_list_404_example, + schema=NotFoundErrorSerializer + ) + } + + code_examples = [ + { + 'lang': 'Bash', + 'source': oembed_list_curl, + }, + ] + + swagger_setup = { + 'operation_id': "oembed_list", + 'operation_description': desc, + 'query_serializer': OembedRequestSerializer, + 'responses': responses, + 'code_examples': code_examples + } diff --git a/openverse-api/catalog/api/docs/media_docs.py b/openverse-api/catalog/api/docs/media_docs.py new file mode 100644 index 000000000..e19afacf0 --- /dev/null +++ b/openverse-api/catalog/api/docs/media_docs.py @@ -0,0 +1,70 @@ +refer_sample = """ +You can refer to the cURL request samples for examples on how to consume this +endpoint. +""" + + +def fields_to_md(field_names): + """ + Create a Markdown representation of the given list of names to use in + Swagger documentation. + + :param field_names: the list of field names to convert to Markdown + :return: the names as a Markdown string + """ + + *all_but_last, last = field_names + all_but_last = ', '.join([f'`{name}`' for name in all_but_last]) + return f'{all_but_last} and `{last}`' + + +class MediaSearch: + desc = ( + """ +Results are ranked in order of relevance and paginated on the basis of the +`page` param. The `page_size` param controls the total number of pages. + +Although there may be millions of relevant records, only the most +relevant several thousand records can be viewed. This is by design: +the search endpoint should be used to find the top 10,000 most relevant +results, not for exhaustive search or bulk download of every barely +relevant result. As such, the caller should not try to access pages +beyond `page_count`, or else the server will reject the query. + +For more precise results, you can go to the +[Openverse Syntax Guide](https://search.creativecommons.org/search-help) +for information about creating queries and +[Apache Lucene Syntax Guide](https://lucene.apache.org/core/2_9_4/queryparsersyntax.html) +for information on structuring advanced searches. +""" # noqa + f'{refer_sample}' + ) + + +class MediaStats: + desc = ( + """ +You can use this endpoint to get details about content providers such as +`source_name`, `display_name`, and `source_url` along with a count of the number +of individual items indexed from them. +""" # noqa + f'{refer_sample}' + ) + + +class MediaDetail: + desc = refer_sample + + +class MediaRelated: + desc = refer_sample + + +class MediaComplain: + desc = ( + """ +By using this endpoint, you can report a file if it infringes copyright, +contains mature or sensitive content and others. +""" # noqa + f'{refer_sample}' + ) From a9f92f434d1ae1c58abe335804d04b166c16eef8 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:57:36 +0530 Subject: [PATCH 14/37] Consolidate endpoints into a DRF viewset --- .../catalog/api/views/audio_views.py | 323 +++-------- .../catalog/api/views/image_views.py | 500 ++++-------------- .../catalog/api/views/media_views.py | 131 ++--- 3 files changed, 205 insertions(+), 749 deletions(-) diff --git a/openverse-api/catalog/api/views/audio_views.py b/openverse-api/catalog/api/views/audio_views.py index 786f81731..36ae1ef0c 100644 --- a/openverse-api/catalog/api/views/audio_views.py +++ b/openverse-api/catalog/api/views/audio_views.py @@ -1,290 +1,84 @@ import logging -from drf_yasg import openapi +from django.utils.decorators import method_decorator from drf_yasg.utils import swagger_auto_schema -from rest_framework.views import APIView -from rest_framework.generics import CreateAPIView +from rest_framework.decorators import action from rest_framework.response import Response -from catalog.api.controllers import search_controller -from catalog.api.examples import ( - audio_search_curl, - audio_search_200_example, - audio_search_400_example, - recommendations_audio_read_curl, - recommendations_audio_read_200_example, - recommendations_audio_read_404_example, - audio_detail_curl, - audio_detail_200_example, - audio_detail_404_example, - audio_stats_curl, - audio_stats_200_example, +from catalog.api.docs.audio_docs import ( + AudioSearch, + AudioStats, + AudioDetail, + AudioRelated, + AudioComplain, ) -from catalog.api.models import Audio, AudioReport -from catalog.api.serializers.media_serializers import ProxiedImageSerializer +from catalog.api.models import Audio from catalog.api.serializers.audio_serializers import ( AudioSearchQueryStringSerializer, - AudioSearchResultsSerializer, AudioSerializer, ReportAudioSerializer, - AboutAudioSerializer, -) -from catalog.api.serializers.error_serializers import ( - InputErrorSerializer, - NotFoundErrorSerializer, -) -from catalog.api.views.media_views import ( - RESULTS, - RESULT_COUNT, - PAGE_COUNT, - fields_to_md, - SearchMedia, - RelatedMedia, - MediaDetail, - MediaStats, - ImageProxy, + AudioWaveformSerializer, ) +from catalog.api.utils.exceptions import get_api_exception +from catalog.api.utils.throttle import OneThousandPerMinute from catalog.api.utils.waveform import ( download_audio, generate_waveform, process_waveform_output, cleanup, ) -from catalog.custom_auto_schema import CustomAutoSchema +from catalog.api.views.media_views import MediaViewSet log = logging.getLogger(__name__) -class SearchAudio(SearchMedia): - audio_search_description = f""" -audio_search is an API endpoint to search audio files using a query string. - -By using this endpoint, you can obtain search results based on specified -query and optionally filter results by -{fields_to_md(AudioSearchQueryStringSerializer.fields_names)}. - -Results are ranked in order of relevance. - -{SearchMedia.search_description}""" # noqa - - audio_search_response = { - "200": openapi.Response( - description="OK", - examples=audio_search_200_example, - schema=AudioSearchResultsSerializer(many=True) - ), - "400": openapi.Response( - description="Bad Request", - examples=audio_search_400_example, - schema=InputErrorSerializer - ), - } - - @swagger_auto_schema(operation_id='audio_search', - operation_description=audio_search_description, - query_serializer=AudioSearchQueryStringSerializer, - responses=audio_search_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': audio_search_curl - } - ]) - def get(self, request, fmt=None): - # Parse and validate query parameters - return self._get( - request, - 'audio', - 'search-qa-audio', - AudioSearchQueryStringSerializer, - AudioSerializer, - AudioSearchResultsSerializer, - ) - - -class RelatedAudio(RelatedMedia): - recommendations_audio_read_description = f""" -recommendations_audio_read is an API endpoint to get related audio files -for a specified audio ID. - -By using this endpoint, you can get the details of related audio such as -{fields_to_md(AudioSerializer.fields_names)}. - -{RelatedMedia.recommendations_read_description}""" # noqa - - recommendations_audio_read_response = { - "200": openapi.Response( - description="OK", - examples=recommendations_audio_read_200_example, - schema=AudioSerializer - ), - "404": openapi.Response( - description="Not Found", - examples=recommendations_audio_read_404_example, - schema=NotFoundErrorSerializer - ) - } - - @swagger_auto_schema(operation_id="recommendations_audio_read", - operation_description=recommendations_audio_read_description, # noqa: E501 - responses=recommendations_audio_read_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': recommendations_audio_read_curl - } - ], - manual_parameters=[ - openapi.Parameter( - 'identifier', openapi.IN_PATH, - "The unique identifier for the audio.", - type=openapi.TYPE_STRING, - required=True - ), - ]) - def get(self, request, identifier, format=None): - related, result_count = search_controller.related_media( - uuid=identifier, - index='audio', - request=request, - filter_dead=True - ) - - context = {'request': request} - serialized_related = AudioSerializer( - related, - many=True, - context=context - ).data - response_data = { - RESULT_COUNT: result_count, - PAGE_COUNT: 0, - RESULTS: serialized_related - } - serialized_response = AudioSearchResultsSerializer(data=response_data) - return Response(status=200, data=serialized_response.initial_data) - - -class ReportAudioView(CreateAPIView): +@method_decorator(swagger_auto_schema(**AudioSearch.swagger_setup), 'list') +@method_decorator(swagger_auto_schema(**AudioStats.swagger_setup), 'stats') +@method_decorator(swagger_auto_schema(**AudioDetail.swagger_setup), 'retrieve') +@method_decorator(swagger_auto_schema(**AudioRelated.swagger_setup), 'related') +@method_decorator(swagger_auto_schema(**AudioComplain.swagger_setup), 'report') +@method_decorator(swagger_auto_schema(auto_schema=None), 'thumbnail') +@method_decorator(swagger_auto_schema(auto_schema=None), 'waveform') +class AudioViewSet(MediaViewSet): + """ + Viewset for all endpoints pertaining to audio. """ - audio_report_create - - audio_report_create is an API endpoint to report an issue about a - specified audio ID to Creative Commons. - - By using this endpoint, you can report an audio file if it infringes - copyright, contains mature or sensitive content and others. - - You can refer to Bash's Request Samples for example on how to use - this endpoint. - """ # noqa - swagger_schema = CustomAutoSchema - queryset = AudioReport.objects.all() - serializer_class = ReportAudioSerializer + model_class = Audio + query_serializer_class = AudioSearchQueryStringSerializer + default_index = 'audio' + qa_index = 'search-qa-audio' -class AudioDetail(MediaDetail): serializer_class = AudioSerializer - queryset = Audio.objects.all() - audio_detail_description = f""" -audio_detail is an API endpoint to get the details of a specified audio ID. - -By using this endpoint, you can get audio details such as -{fields_to_md(AudioSerializer.fields_names)}. - -{MediaDetail.detail_description}""" # noqa - - audio_detail_response = { - "200": openapi.Response( - description="OK", - examples=audio_detail_200_example, - schema=AudioSerializer), - "404": openapi.Response( - description="OK", - examples=audio_detail_404_example, - schema=NotFoundErrorSerializer - ) - } - - @swagger_auto_schema(operation_id='audio_detail', - operation_description=audio_detail_description, - responses=audio_detail_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': audio_detail_curl, - } - ]) - def get(self, request, identifier, format=None): - """ Get the details of a single audio file. """ - return self.retrieve(request, identifier) + # Extra actions -class AudioStats(MediaStats): - audio_stats_description = f""" -audio_stats is an API endpoint to get a list of all content providers and their -respective number of audio files in the Openverse catalog. + @action(detail=True, + url_path='thumb', + url_name='thumb', + throttle_classes=[OneThousandPerMinute]) + def thumbnail(self, request, *_, **__): + audio = self.get_object() -{MediaStats.media_stats_description}""" # noqa - - audio_stats_response = { - "200": openapi.Response( - description="OK", - examples=audio_stats_200_example, - schema=AboutAudioSerializer(many=True) - ) - } - - @swagger_auto_schema(operation_id='audio_stats', - operation_description=audio_stats_description, - responses=audio_stats_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': audio_stats_curl, - } - ]) - def get(self, request, format=None): - return self._get(request, 'audio') - - -class AudioArt(ImageProxy): - """ - Return the thumbnail of the artwork of the audio. This returns the thumbnail - of the audio, falling back to the thumbnail of the audio set. - """ - - queryset = Audio.objects.all() - - def get(self, request, identifier, format=None): - serialized = ProxiedImageSerializer(data=request.data) - serialized.is_valid() - try: - audio = Audio.objects.get(identifier=identifier) - image_url = audio.thumbnail - if not image_url: - image_url = audio.audio_set.url - except Audio.DoesNotExist: - return Response(status=404, data='Audio not found') - except AttributeError: - return Response(status=404, data='Audio set not found') + image_url = None + if thumbnail := audio.thumbnail: + image_url = thumbnail + elif audio.audio_set and (thumbnail := audio.audio_set.url): + image_url = thumbnail if not image_url: - return Response(status=404, data='Cover art URL not found') + raise get_api_exception('Could not find artwork.', 404) - if serialized.data['full_size']: - return self._get(image_url, None) + is_full_size = request.query_params.get('full_size', False) + if is_full_size: + return self._get_proxied_image(image_url, None) else: - return self._get(image_url) - + return self._get_proxied_image(image_url) -class AudioWaveform(APIView): - swagger_schema = None - - def get(self, request, identifier, format=None): - try: - audio = Audio.objects.get(identifier=identifier) - except Audio.DoesNotExist: - return Response(status=404, data='Audio not found') + @action(detail=True, + serializer_class=AudioWaveformSerializer, + throttle_classes=[OneThousandPerMinute]) + def waveform(self, *_, **__): + audio = self.get_object() file_name = None try: @@ -292,15 +86,18 @@ def get(self, request, identifier, format=None): awf_out = generate_waveform(file_name, audio.duration) data = process_waveform_output(awf_out) - return Response(status=200, data={ - 'len': len(data), - 'points': data, - }) + obj = {'points': data} + serializer = self.get_serializer(obj) + + return Response(status=200, data=serializer.data) except Exception as e: - return Response(status=500, data={ - 'message': "It's not you, it's me.", - 'error': str(e) - }) + raise get_api_exception(getattr(e, 'message', str(e))) finally: if file_name is not None: cleanup(file_name) + + @action(detail=True, + methods=['post'], + serializer_class=ReportAudioSerializer) + def report(self, *args, **kwargs): + return self._report(*args, **kwargs) diff --git a/openverse-api/catalog/api/views/image_views.py b/openverse-api/catalog/api/views/image_views.py index 4b83bf961..5977e2eae 100644 --- a/openverse-api/catalog/api/views/image_views.py +++ b/openverse-api/catalog/api/views/image_views.py @@ -4,266 +4,128 @@ import libxmp import piexif import requests -from PIL import Image as img -from django.urls import reverse -from django.http.response import HttpResponse, FileResponse -from drf_yasg import openapi +from PIL import Image as PILImage +from django.conf import settings +from django.http.response import HttpResponse, FileResponse, Http404 +from django.utils.decorators import method_decorator from drf_yasg.utils import swagger_auto_schema -from rest_framework import status -from rest_framework.generics import GenericAPIView, CreateAPIView +from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.views import APIView -import catalog.api.controllers.search_controller as search_controller -from catalog.api.examples import ( - image_search_curl, - image_search_200_example, - image_search_400_example, - recommendations_images_read_curl, - recommendations_images_read_200_example, - recommendations_images_read_404_example, - image_detail_curl, - image_detail_200_example, - image_detail_404_example, - oembed_list_200_example, - oembed_list_404_example, - image_stats_curl, - image_stats_200_example, - report_image_curl, - images_report_create_201_example, +from catalog.api.docs.image_docs import ( + ImageSearch, + ImageStats, + ImageDetail, + ImageRelated, + ImageOembed, + ImageComplain, ) -from catalog.api.models import Image, ImageReport -from catalog.api.serializers.error_serializers import ( - InputErrorSerializer, - NotFoundErrorSerializer, -) -from catalog.api.serializers.media_serializers import ProxiedImageSerializer +from catalog.api.models import Image from catalog.api.serializers.image_serializers import ( ImageSearchQueryStringSerializer, - ImageSearchResultsSerializer, ImageSerializer, ReportImageSerializer, WatermarkQueryStringSerializer, + OembedRequestSerializer, OembedSerializer, - OembedResponseSerializer, - AboutImageSerializer, ) from catalog.api.utils import ccrel -from catalog.api.utils.exceptions import input_error_response +from catalog.api.utils.exceptions import get_api_exception +from catalog.api.utils.throttle import OneThousandPerMinute from catalog.api.utils.watermark import watermark -from catalog.api.views.media_views import ( - refer_sample, - RESULTS, - RESULT_COUNT, - PAGE_COUNT, - fields_to_md, - SearchMedia, - RelatedMedia, - MediaDetail, - MediaStats, - ImageProxy, -) -from catalog.custom_auto_schema import CustomAutoSchema +from catalog.api.views.media_views import MediaViewSet log = logging.getLogger(__name__) -class SearchImages(SearchMedia): - image_search_description = f""" -image_search is an API endpoint to search images using a query string. - -By using this endpoint, you can obtain search results based on specified query -and optionally filter results by -{fields_to_md(ImageSearchQueryStringSerializer.fields_names)}. - -Results are ranked in order of relevance. - -{SearchMedia.search_description}""" # noqa - - image_search_response = { - "200": openapi.Response( - description="OK", - examples=image_search_200_example, - schema=ImageSearchResultsSerializer(many=True) - ), - "400": openapi.Response( - description="Bad Request", - examples=image_search_400_example, - schema=InputErrorSerializer - ) - } - - @swagger_auto_schema(operation_id='image_search', - operation_description=image_search_description, - query_serializer=ImageSearchQueryStringSerializer, - responses=image_search_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': image_search_curl - } - ]) - def get(self, request, format=None): - # Parse and validate query parameters - return self._get( - request, - 'image', - 'search-qa-image', - ImageSearchQueryStringSerializer, - ImageSerializer, - ImageSearchResultsSerializer, - ) - - -class RelatedImage(RelatedMedia): - recommendations_images_read_description = f""" -recommendations_images_read is an API endpoint to get related images -for a specified image ID. - -By using this endpoint, you can get the details of related images such as -{fields_to_md(ImageSerializer.fields_names)}. - -{RelatedMedia.recommendations_read_description}""" # noqa - - recommendations_images_read_response = { - "200": openapi.Response( - description="OK", - examples=recommendations_images_read_200_example, - schema=ImageSerializer - ), - "404": openapi.Response( - description="Not Found", - examples=recommendations_images_read_404_example, - schema=NotFoundErrorSerializer - ) - } - - @swagger_auto_schema(operation_id="recommendations_images_read", - operation_description=recommendations_images_read_description, # noqa: E501 - responses=recommendations_images_read_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': recommendations_images_read_curl - } - ], - manual_parameters=[ - openapi.Parameter( - 'identifier', openapi.IN_PATH, - "The unique identifier for the image.", - type=openapi.TYPE_STRING, - required=True - ), - ]) - def get(self, request, identifier, format=None): - related, result_count = search_controller.related_media( - uuid=identifier, - index='image', - request=request, - filter_dead=True - ) - - context = {'request': request} - serialized_related = ImageSerializer( - related, - many=True, - context=context - ).data - response_data = { - RESULT_COUNT: result_count, - PAGE_COUNT: 0, - RESULTS: serialized_related - } - serialized_response = ImageSearchResultsSerializer(data=response_data) - return Response(status=200, data=serialized_response.initial_data) +@method_decorator(swagger_auto_schema(**ImageSearch.swagger_setup), 'list') +@method_decorator(swagger_auto_schema(**ImageStats.swagger_setup), 'stats') +@method_decorator(swagger_auto_schema(**ImageDetail.swagger_setup), 'retrieve') +@method_decorator(swagger_auto_schema(**ImageRelated.swagger_setup), 'related') +@method_decorator(swagger_auto_schema(**ImageComplain.swagger_setup), 'report') +@method_decorator(swagger_auto_schema(**ImageOembed.swagger_setup), 'oembed') +@method_decorator(swagger_auto_schema(auto_schema=None), 'thumbnail') +@method_decorator(swagger_auto_schema(auto_schema=None), 'watermark') +class ImageViewSet(MediaViewSet): + """ + Viewset for all endpoints pertaining to images. + """ + model_class = Image + query_serializer_class = ImageSearchQueryStringSerializer + default_index = 'image' + qa_index = 'search-qa-image' -class ImageDetail(MediaDetail): serializer_class = ImageSerializer - queryset = Image.objects.all() - image_detail_description = f""" -image_detail is an API endpoint to get the details of a specified image ID. -By using this endpoint, you can image details such as -{fields_to_md(ImageSerializer.fields_names)}. + # Extra actions -{MediaDetail.detail_description}""" # noqa + @action(detail=False, + url_path='oembed', + url_name='oembed', + serializer_class=OembedSerializer) + def oembed(self, request, *_, **__): + params = OembedRequestSerializer(data=request.query_params) + params.is_valid(raise_exception=True) - image_detail_response = { - "200": openapi.Response( - description="OK", - examples=image_detail_200_example, - schema=ImageSerializer - ), - "404": openapi.Response( - description='Not Found', - examples=image_detail_404_example, - schema=NotFoundErrorSerializer - ) - } + context = self.get_serializer_context() - @swagger_auto_schema(operation_id="image_detail", - operation_description=image_detail_description, - responses=image_detail_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': image_detail_curl - } - ]) - def get(self, request, identifier, format=None): - """ Get the details of a single image. """ - resp = self.retrieve(request, identifier) - # Proxy insecure HTTP images at full resolution. - if 'http://' in resp.data[search_controller.URL]: - secure = request.build_absolute_uri( - reverse('image-thumb', kwargs={'identifier': identifier}) - ) - secure += '?full_size=True' - resp.data[search_controller.URL] = secure + url = params.validated_data['url'] + identifier = url.rsplit('/', 1)[1] + try: + image = self.get_queryset().get(identifier=identifier) + except Image.DoesNotExist: + return get_api_exception('Could not find image.', 404) + if not (image.height and image.width): + image_file = requests.get(image.url) + width, height = PILImage.open(io.BytesIO(image_file.content)).size + context |= { + 'width': width, + 'height': height, + } - return resp + serializer = self.get_serializer(image, context=context) + return Response(data=serializer.data) + @action(detail=True, + url_path='thumb', + url_name='thumb', + throttle_classes=[OneThousandPerMinute]) + def thumbnail(self, request, *_, **__): + image = self.get_object() -def _save_wrapper(pil_img, exif_bytes, destination): - """ - PIL crashes if exif_bytes=None, so we have to wrap it to avoid littering - the code with branches. - """ - if exif_bytes: - pil_img.save(destination, 'jpeg', exif=exif_bytes) - else: - pil_img.save(destination, 'jpeg') + image_url = image.url + if not image_url: + raise get_api_exception('Could not find image.', 404) + is_full_size = request.query_params.get('full_size', False) + if is_full_size: + return self._get_proxied_image(image_url, None) + else: + return self._get_proxied_image(image_url) -class Watermark(GenericAPIView): - """ - Given an image identifier as a URL parameter, produce an attribution - watermark. This entails drawing a frame around the image and embedding - ccREL metadata inside of the file. - """ - lookup_field = 'identifier' - serializer_class = WatermarkQueryStringSerializer + @action(detail=True, + url_path='watermark', + url_name='watermark') + def watermark(self, request, *_, **__): + if not settings.WATERMARK_ENABLED: + raise Http404 # watermark feature is disabled - @swagger_auto_schema(query_serializer=WatermarkQueryStringSerializer) - def get(self, request, identifier, format=None): params = WatermarkQueryStringSerializer(data=request.query_params) - if not params.is_valid(): - return input_error_response() - try: - image_record = Image.objects.get(identifier=identifier) - except Image.DoesNotExist: - return Response(status=404, data='Not Found') - image_url = str(image_record.url) + params.is_valid(raise_exception=True) + + image = self.get_object() + image_url = image.url image_info = { - 'title': image_record.title, - 'creator': image_record.creator, - 'license': image_record.license, - 'license_version': image_record.license_version + attr: getattr(image, attr) + for attr in ['title', 'creator', 'license', 'license_version'] } + # Create the actual watermarked image. watermarked, exif = watermark( - image_url, image_info, params.data['watermark'] + image_url, + image_info, + params.data['watermark'] ) # Re-insert EXIF metadata. if exif: @@ -271,185 +133,45 @@ def get(self, request, identifier, format=None): else: exif_bytes = None img_bytes = io.BytesIO() - _save_wrapper(watermarked, exif_bytes, img_bytes) + self._save_wrapper(watermarked, exif_bytes, img_bytes) + if params.data['embed_metadata']: # Embed ccREL metadata with XMP. work_properties = { - 'creator': image_record.creator, - 'license_url': image_record.license_url, - 'attribution': image_record.attribution, - 'work_landing_page': image_record.foreign_landing_url, - 'identifier': str(image_record.identifier) + 'creator': image.creator, + 'license_url': image.license_url, + 'attribution': image.attribution, + 'work_landing_page': image.foreign_landing_url, + 'identifier': str(image.identifier) } try: with_xmp = ccrel.embed_xmp_bytes(img_bytes, work_properties) return FileResponse(with_xmp, content_type='image/jpeg') except (libxmp.XMPError, AttributeError) as e: - # Just send the EXIF-ified file if libxmp fails to add metadata. - log.error( - f'Failed to add XMP metadata to {image_record.identifier}' - ) - log.error(e) + # Just send the EXIF-ified file if libxmp fails to add metadata response = HttpResponse(content_type='image/jpeg') - _save_wrapper(watermarked, exif_bytes, response) + self._save_wrapper(watermarked, exif_bytes, response) return response else: response = HttpResponse(img_bytes, content_type='image/jpeg') - _save_wrapper(watermarked, exif_bytes, response) + self._save_wrapper(watermarked, exif_bytes, response) return response + @action(detail=True, + methods=['post'], + serializer_class=ReportImageSerializer) + def report(self, *args, **kwargs): + return super().report(*args, **kwargs) -class OembedView(APIView): - swagger_schema = CustomAutoSchema - oembed_list_description = \ - """ - oembed_list is an API endpoint to retrieve embedded content from a - specified image URL. - - By using this endpoint, you can retrieve embedded content such as - `version`, `type`, `width`, `height`, `title`, `author_name`, - `author_url`, and `license_url`. - - You can refer to Bash's Request Samples for example on how to use - this endpoint. - """ # noqa - oembed_list_response = { - "200": openapi.Response( - description="OK", - examples=oembed_list_200_example, - schema=OembedResponseSerializer - ), - "404": openapi.Response( - description="Not Found", - examples=oembed_list_404_example, - schema=NotFoundErrorSerializer - ) - } + # Helper functions - oembed_list_bash = \ + @staticmethod + def _save_wrapper(pil_img, exif_bytes, destination): """ - # Retrieve embedded content from image URL (https://ccsearch.creativecommons.org/photos/7c829a03-fb24-4b57-9b03-65f43ed19395) - curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/oembed?url=https://ccsearch.creativecommons.org/photos/7c829a03-fb24-4b57-9b03-65f43ed19395 - """ # noqa - - @swagger_auto_schema(operation_id="oembed_list", - operation_description=oembed_list_description, - query_serializer=OembedSerializer, - responses=oembed_list_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': oembed_list_bash - } - ]) - def get(self, request): - url = request.query_params.get('url', '') - - if not url: - return Response(status=404, data='Not Found') - try: - identifier = url.rsplit('/', 1)[1] - image_record = Image.objects.get(identifier=identifier) - except Image.DoesNotExist: - return Response(status=404, data='Not Found') - if not image_record.height or image_record.width: - image = requests.get(image_record.url) - width, height = img.open(io.BytesIO(image.content)).size - else: - width, height = image_record.width, image_record.height - resp = { - 'version': 1.0, - 'type': 'photo', - 'width': width, - 'height': height, - 'title': image_record.title, - 'author_name': image_record.creator, - 'author_url': image_record.creator_url, - 'license_url': image_record.license_url - } - - return Response(data=resp, status=status.HTTP_200_OK) - - -class ReportImageView(CreateAPIView): - report_image_description = f""" -images_report_create is an API endpoint to report an issue about a specified -image ID to Openverse. - -By using this endpoint, you can report an image if it infringes copyright, -contains mature or sensitive content and others. - -{refer_sample}""" # noqa - - swagger_schema = CustomAutoSchema - queryset = ImageReport.objects.all() - serializer_class = ReportImageSerializer - - @swagger_auto_schema(operation_id='images_report_create', - operation_description=report_image_description, - query_serializer=ReportImageSerializer, - responses={ - "201": openapi.Response( - description="OK", - examples=images_report_create_201_example, - schema=ReportImageSerializer - ) - }, - code_examples=[ - { - 'lang': 'Bash', - 'source': report_image_curl, - } - ]) - def post(self, request, *args, **kwargs): - return super(ReportImageView, self).post(request, *args, **kwargs) - - -class ImageStats(MediaStats): - image_stats_description = f""" -image_stats is an API endpoint to get a list of all content providers and their -respective number of images in the Openverse catalog. - -{MediaStats.media_stats_description}""" # noqa - - image_stats_response = { - "200": openapi.Response( - description="OK", - examples=image_stats_200_example, - schema=AboutImageSerializer(many=True) - ) - } - - @swagger_auto_schema(operation_id='image_stats', - operation_description=image_stats_description, - responses=image_stats_response, - code_examples=[ - { - 'lang': 'Bash', - 'source': image_stats_curl, - } - ]) - def get(self, request, format=None): - return self._get(request, 'image') - - -class ProxiedImage(ImageProxy): - """ - Return the thumb of an image. - """ - - queryset = Image.objects.all() - - def get(self, request, identifier, format=None): - serialized = ProxiedImageSerializer(data=request.data) - serialized.is_valid() - try: - image = Image.objects.get(identifier=identifier) - image_url = image.url - except Image.DoesNotExist: - return Response(status=404, data='Not Found') - - if serialized.data['full_size']: - return self._get(image_url, None) + PIL crashes if exif_bytes=None, so we have to wrap it to avoid littering + the code with branches. + """ + if exif_bytes: + pil_img.save(destination, 'jpeg', exif=exif_bytes) else: - return self._get(image_url) + pil_img.save(destination, 'jpeg') diff --git a/openverse-api/catalog/api/views/media_views.py b/openverse-api/catalog/api/views/media_views.py index e100e1b7c..dfec53092 100644 --- a/openverse-api/catalog/api/views/media_views.py +++ b/openverse-api/catalog/api/views/media_views.py @@ -1,110 +1,21 @@ -import logging +from urllib.error import HTTPError +from urllib.request import urlopen from django.conf import settings from django.http.response import HttpResponse +from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.viewsets import ReadOnlyModelViewSet -from urllib.error import HTTPError -from urllib.request import urlopen from catalog.api.controllers import search_controller from catalog.api.controllers.search_controller import get_sources -from catalog.api.serializers.provider_serializers import ProviderSerializer -from catalog.api.utils.pagination import StandardPagination from catalog.api.models import ContentProvider +from catalog.api.serializers.provider_serializers import ProviderSerializer from catalog.api.utils.exceptions import get_api_exception +from catalog.api.utils.pagination import StandardPagination from catalog.custom_auto_schema import CustomAutoSchema -log = logging.getLogger(__name__) - -refer_sample = """ -You can refer to the cURL request samples for examples on how to consume this -endpoint. -""" - - -def fields_to_md(field_names): - """ - Create a Markdown representation of the given list of names to use in - Swagger documentation. - - :param field_names: the list of field names to convert to Markdown - :return: the names as a Markdown string - """ - - *all_but_last, last = field_names - all_but_last = ', '.join([f'`{name}`' for name in all_but_last]) - return f'{all_but_last} and `{last}`' - - -def _get_user_ip(request): - """ - Read request headers to find the correct IP address. - It is assumed that X-Forwarded-For has been sanitized by the load balancer - and thus cannot be rewritten by malicious users. - :param request: A Django request object. - :return: An IP address. - """ - x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') - if x_forwarded_for: - ip = x_forwarded_for.split(',')[0] - else: - ip = request.META.get('REMOTE_ADDR') - return ip - - -class MediaSearch: - desc = ( - """ -Results are ranked in order of relevance and paginated on the basis of the -`page` param. The `page_size` param controls the total number of pages. - -Although there may be millions of relevant records, only the most -relevant several thousand records can be viewed. This is by design: -the search endpoint should be used to find the top 10,000 most relevant -results, not for exhaustive search or bulk download of every barely -relevant result. As such, the caller should not try to access pages -beyond `page_count`, or else the server will reject the query. - -For more precise results, you can go to the -[Openverse Syntax Guide](https://search.creativecommons.org/search-help) -for information about creating queries and -[Apache Lucene Syntax Guide](https://lucene.apache.org/core/2_9_4/queryparsersyntax.html) -for information on structuring advanced searches. -""" # noqa - f'{refer_sample}' - ) - - -class MediaStats: - desc = ( - """ -You can use this endpoint to get details about content providers such as -`source_name`, `display_name`, and `source_url` along with a count of the number -of individual items indexed from them. -""" # noqa - f'{refer_sample}' - ) - - -class MediaDetail: - desc = refer_sample - - -class MediaRelated: - desc = refer_sample - - -class MediaComplain: - desc = ( - """ -By using this endpoint, you can report a file if it infringes copyright, -contains mature or sensitive content and others. -""" # noqa - f'{refer_sample}' - ) - class MediaViewSet(ReadOnlyModelViewSet): swagger_schema = CustomAutoSchema @@ -145,10 +56,9 @@ def list(self, request, *_, **__): page = self.paginator.page params = self.query_serializer_class(data=request.query_params) - if not params.is_valid(): - raise get_api_exception('Input is invalid.', 400) + params.is_valid(raise_exception=True) - hashed_ip = hash(_get_user_ip(request)) + hashed_ip = hash(self._get_user_ip(request)) qa = params.validated_data['qa'] filter_dead = params.validated_data['filter_dead'] @@ -205,8 +115,35 @@ def related(self, request, identifier=None, *_, **__): serializer = self.get_serializer(results, many=True) return self.get_paginated_response(serializer.data) + def report(self, request, *_, **__): + media = self.get_object() + identifier = media.identifier + serializer = self.get_serializer(data=request.data) + if not serializer.is_valid(): + raise get_api_exception('Invalid input.', 400) + report = serializer.save(identifier=identifier) + + serializer = self.get_serializer(report) + return Response(data=serializer.data, status=status.HTTP_201_CREATED) + # Helper functions + @staticmethod + def _get_user_ip(request): + """ + Read request headers to find the correct IP address. + It is assumed that X-Forwarded-For has been sanitized by the load balancer + and thus cannot be rewritten by malicious users. + :param request: A Django request object. + :return: An IP address. + """ + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + if x_forwarded_for: + ip = x_forwarded_for.split(',')[0] + else: + ip = request.META.get('REMOTE_ADDR') + return ip + @staticmethod def _get_proxied_image(image_url, width=settings.THUMBNAIL_WIDTH_PX): if width is None: # full size From 0acf192837aaca7a0c71cd416fe5e75110a368c3 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:57:55 +0530 Subject: [PATCH 15/37] Use DRF router to automatically define endpoints from the viewset --- openverse-api/catalog/urls/__init__.py | 22 +++++------- openverse-api/catalog/urls/audio.py | 43 ---------------------- openverse-api/catalog/urls/images.py | 49 -------------------------- 3 files changed, 8 insertions(+), 106 deletions(-) delete mode 100644 openverse-api/catalog/urls/audio.py delete mode 100644 openverse-api/catalog/urls/images.py diff --git a/openverse-api/catalog/urls/__init__.py b/openverse-api/catalog/urls/__init__.py index b066f181c..2c036e586 100644 --- a/openverse-api/catalog/urls/__init__.py +++ b/openverse-api/catalog/urls/__init__.py @@ -21,14 +21,14 @@ from django.views.generic import RedirectView from drf_yasg import openapi from drf_yasg.views import get_schema_view +from rest_framework.routers import SimpleRouter -from catalog.api.views.image_views import Watermark, OembedView +from catalog.api.views.audio_views import AudioViewSet +from catalog.api.views.image_views import ImageViewSet from catalog.api.views.site_views import HealthCheck, CheckRates from catalog.api.utils.status_code_view import get_status_code_view from catalog.urls.auth_tokens import urlpatterns as auth_tokens_patterns -from catalog.urls.audio import urlpatterns as audio_patterns -from catalog.urls.images import urlpatterns as images_patterns description = """ # Introduction @@ -193,16 +193,14 @@ 'reason': 'This API endpoint has been discontinued.' } +router = SimpleRouter() +router.register('audio', AudioViewSet, basename='audio') +router.register('images', ImageViewSet, basename='image') + versioned_paths = [ path('rate_limit', CheckRates.as_view(), name='key_info'), path('auth_tokens/', include(auth_tokens_patterns)), - # Audio - path('audio/', include(audio_patterns)), - - # Images - path('images/', include(images_patterns)), - # Deprecated path( 'sources', @@ -235,11 +233,7 @@ get_status_code_view(discontinuation_message, 410).as_view(), name='make-link' ), -] -if settings.WATERMARK_ENABLED: - versioned_paths.append( - path('watermark/', Watermark.as_view()) - ) +] + router.urls urlpatterns = [ path('', RedirectView.as_view(pattern_name='root')), diff --git a/openverse-api/catalog/urls/audio.py b/openverse-api/catalog/urls/audio.py deleted file mode 100644 index b103fc2f8..000000000 --- a/openverse-api/catalog/urls/audio.py +++ /dev/null @@ -1,43 +0,0 @@ -from django.urls import path - -from catalog.api.views.audio_views import ( - SearchAudio, - AudioDetail, - RelatedAudio, - AudioStats, - AudioArt, - AudioWaveform, -) - -urlpatterns = [ - path( - 'stats', - AudioStats.as_view(), - name='audio-stats' - ), - path( - '', - AudioDetail.as_view(), - name='audio-detail' - ), - path( - '/thumb', - AudioArt.as_view(), - name='audio-thumb' - ), - path( - '/recommendations', - RelatedAudio.as_view(), - name='audio-related' - ), - path( - '/waveform', - AudioWaveform.as_view(), - name='audio-waveform' - ), - path( - '', - SearchAudio.as_view(), - name='audio' - ), -] diff --git a/openverse-api/catalog/urls/images.py b/openverse-api/catalog/urls/images.py deleted file mode 100644 index a0de32f4a..000000000 --- a/openverse-api/catalog/urls/images.py +++ /dev/null @@ -1,49 +0,0 @@ -from django.urls import path - -from catalog.api.views.image_views import ( - SearchImages, - ImageDetail, - RelatedImage, - ImageStats, - ReportImageView, - ProxiedImage, - OembedView, -) - -urlpatterns = [ - path( - 'stats', - ImageStats.as_view(), - name='image-stats' - ), - path( - 'oembed', - OembedView.as_view(), - name='image-oembed' - ), - path( - '', - ImageDetail.as_view(), - name='image-detail' - ), - path( - '/thumb', - ProxiedImage.as_view(), - name='image-thumb' - ), - path( - '/recommendations', - RelatedImage.as_view(), - name='image-related' - ), - path( - '/report', - ReportImageView.as_view(), - name='report-image' - ), - path( - '', - SearchImages.as_view(), - name='images' - ), -] From d5e050f2ba6d7a637cc3bc246b98ebe28c2b054a Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:58:06 +0530 Subject: [PATCH 16/37] Update tests based on the new code structure --- openverse-api/test/audio_integration_test.py | 2 +- openverse-api/test/backwards_compat_test.py | 16 ++++++++++------ openverse-api/test/image_integration_test.py | 2 +- openverse-api/test/media_integration.py | 2 +- openverse-api/test/v1_integration_test.py | 13 +++++++------ 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/openverse-api/test/audio_integration_test.py b/openverse-api/test/audio_integration_test.py index aa378bfc8..3b8f17324 100644 --- a/openverse-api/test/audio_integration_test.py +++ b/openverse-api/test/audio_integration_test.py @@ -50,7 +50,7 @@ def test_audio_detail(audio_fixture): def test_audio_stats(): - stats('audio', 'audio_count') + stats('audio') @pytest.mark.skip(reason='No images have audio set image yet') diff --git a/openverse-api/test/backwards_compat_test.py b/openverse-api/test/backwards_compat_test.py index e09ee4f35..ad7869cb7 100644 --- a/openverse-api/test/backwards_compat_test.py +++ b/openverse-api/test/backwards_compat_test.py @@ -6,6 +6,8 @@ Run with the `pytest -s` command from this directory. """ +import uuid + import requests from test.constants import API_URL @@ -19,18 +21,19 @@ def test_old_stats_endpoint(): ) assert response.status_code == 301 assert response.is_permanent_redirect - assert response.headers.get('Location') == '/v1/images/stats' + assert response.headers.get('Location') == '/v1/images/stats/' def test_old_related_images_endpoint(): + idx = uuid.uuid4() response = requests.get( - f'{API_URL}/v1/recommendations/images/xyz', + f'{API_URL}/v1/recommendations/images/{idx}', allow_redirects=False, verify=False ) assert response.status_code == 301 assert response.is_permanent_redirect - assert response.headers.get('Location') == '/v1/images/xyz/recommendations' + assert response.headers.get('Location') == f'/v1/images/{idx}/related/' def test_old_oembed_endpoint(): @@ -41,15 +44,16 @@ def test_old_oembed_endpoint(): ) assert response.status_code == 301 assert response.is_permanent_redirect - assert response.headers.get('Location') == '/v1/images/oembed?key=value' + assert response.headers.get('Location') == '/v1/images/oembed/?key=value' def test_old_thumbs_endpoint(): + idx = uuid.uuid4() response = requests.get( - f'{API_URL}/v1/thumbs/xyz', + f'{API_URL}/v1/thumbs/{idx}', allow_redirects=False, verify=False ) assert response.status_code == 301 assert response.is_permanent_redirect - assert response.headers.get('Location') == '/v1/images/xyz/thumb' \ No newline at end of file + assert response.headers.get('Location') == f'/v1/images/{idx}/thumb/' diff --git a/openverse-api/test/image_integration_test.py b/openverse-api/test/image_integration_test.py index 5d2098269..9addb9df6 100644 --- a/openverse-api/test/image_integration_test.py +++ b/openverse-api/test/image_integration_test.py @@ -51,7 +51,7 @@ def test_image_detail(image_fixture): def test_image_stats(): - stats('images', 'image_count') + stats('images') def test_image_thumb(image_fixture): diff --git a/openverse-api/test/media_integration.py b/openverse-api/test/media_integration.py index fe7a2f664..672b7727e 100644 --- a/openverse-api/test/media_integration.py +++ b/openverse-api/test/media_integration.py @@ -60,7 +60,7 @@ def detail(media_type, fixture): assert response.status_code == 200 -def stats(media_type, count_key): +def stats(media_type, count_key='media_count'): response = requests.get(f'{API_URL}/v1/{media_type}/stats', verify=False) parsed_response = json.loads(response.text) assert response.status_code == 200 diff --git a/openverse-api/test/v1_integration_test.py b/openverse-api/test/v1_integration_test.py index b6af528f3..0e8933aff 100644 --- a/openverse-api/test/v1_integration_test.py +++ b/openverse-api/test/v1_integration_test.py @@ -449,14 +449,15 @@ def test_related_image_search_page_consistency( assert len(related['results']) == 10 -def test_report_endpoint(): - identifier = 'dac5f6b0-e07a-44a0-a444-7f43d71f9beb' +def test_report_endpoint(image_fixture): + identifier = image_fixture['results'][0]['id'] payload = { - 'identifier': identifier, - 'reason': 'mature' + 'reason': 'mature', } response = requests.post( - f'{API_URL}/v1/images/{identifier}/report', + f'{API_URL}/v1/images/{identifier}/report/', json=payload, verify=False) assert response.status_code == 201 - return json.loads(response.text) + data = json.loads(response.text) + assert data['identifier'] == identifier + return data From c1522807c2b23da3514823bed2a4cba3f723e9c8 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 00:58:47 +0530 Subject: [PATCH 17/37] Fix code style violations --- openverse-api/catalog/api/examples/audio_requests.py | 1 - openverse-api/catalog/api/examples/image_requests.py | 2 +- openverse-api/catalog/api/views/media_views.py | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/openverse-api/catalog/api/examples/audio_requests.py b/openverse-api/catalog/api/examples/audio_requests.py index b62805987..9074556e3 100644 --- a/openverse-api/catalog/api/examples/audio_requests.py +++ b/openverse-api/catalog/api/examples/audio_requests.py @@ -46,4 +46,3 @@ # Report an issue about audio ID 7c829a03-fb24-4b57-9b03-65f43ed19395 curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "description": "This audio contains sensitive content"}' https://api.openverse.engineering/v1/audio/7c829a03-fb24-4b57-9b03-65f43ed19395/report """ # noqa - diff --git a/openverse-api/catalog/api/examples/image_requests.py b/openverse-api/catalog/api/examples/image_requests.py index b885589a0..906a82475 100644 --- a/openverse-api/catalog/api/examples/image_requests.py +++ b/openverse-api/catalog/api/examples/image_requests.py @@ -50,4 +50,4 @@ oembed_list_curl = """ # Retrieve embedded content from image URL (https://wordpress.org/openverse/photos/7c829a03-fb24-4b57-9b03-65f43ed19395) curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" https://api.openverse.engineering/v1/oembed/?url=https://wordpress.org/openverse/photos/7c829a03-fb24-4b57-9b03-65f43ed19395 -""" # noqa \ No newline at end of file +""" # noqa diff --git a/openverse-api/catalog/api/views/media_views.py b/openverse-api/catalog/api/views/media_views.py index dfec53092..490c4ea14 100644 --- a/openverse-api/catalog/api/views/media_views.py +++ b/openverse-api/catalog/api/views/media_views.py @@ -132,8 +132,8 @@ def report(self, request, *_, **__): def _get_user_ip(request): """ Read request headers to find the correct IP address. - It is assumed that X-Forwarded-For has been sanitized by the load balancer - and thus cannot be rewritten by malicious users. + It is assumed that X-Forwarded-For has been sanitized by the load + balancer and thus cannot be rewritten by malicious users. :param request: A Django request object. :return: An IP address. """ From 796cd10049544f3a71779d52ed5dc133df8d8097 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 01:35:32 +0530 Subject: [PATCH 18/37] Remove redundant import --- openverse-api/catalog/api/views/media_views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openverse-api/catalog/api/views/media_views.py b/openverse-api/catalog/api/views/media_views.py index 490c4ea14..1c00b9e4b 100644 --- a/openverse-api/catalog/api/views/media_views.py +++ b/openverse-api/catalog/api/views/media_views.py @@ -9,7 +9,6 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from catalog.api.controllers import search_controller -from catalog.api.controllers.search_controller import get_sources from catalog.api.models import ContentProvider from catalog.api.serializers.provider_serializers import ProviderSerializer from catalog.api.utils.exceptions import get_api_exception @@ -86,7 +85,7 @@ def list(self, request, *_, **__): @action(detail=False, serializer_class=ProviderSerializer) def stats(self, *_, **__): - source_counts = get_sources(self.default_index) + source_counts = search_controller.get_sources(self.default_index) context = self.get_serializer_context() | { 'source_counts': source_counts, } From a6fa43a3e9233fdbd8465af518a296e1260bb2ed Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 01:43:56 +0530 Subject: [PATCH 19/37] Make serializer nomenclature consistent --- openverse-api/catalog/api/docs/audio_docs.py | 16 +++---- openverse-api/catalog/api/docs/image_docs.py | 16 +++---- .../api/serializers/audio_serializers.py | 14 +++--- .../api/serializers/image_serializers.py | 44 +++++++++---------- .../api/serializers/media_serializers.py | 4 +- .../catalog/api/views/audio_views.py | 8 ++-- .../catalog/api/views/image_views.py | 12 ++--- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/openverse-api/catalog/api/docs/audio_docs.py b/openverse-api/catalog/api/docs/audio_docs.py index fd06ec76e..785004007 100644 --- a/openverse-api/catalog/api/docs/audio_docs.py +++ b/openverse-api/catalog/api/docs/audio_docs.py @@ -24,10 +24,10 @@ audio_report_create_201_example, ) from catalog.api.serializers.audio_serializers import ( - AudioSearchQueryStringSerializer, - AudioSearchResultsSerializer, + AudioSearchRequestSerializer, + AudioSearchSerializer, AudioSerializer, - ReportAudioSerializer, + AudioReportSerializer, ) from catalog.api.serializers.error_serializers import ( InputErrorSerializer, @@ -42,7 +42,7 @@ class AudioSearch(MediaSearch): By using this endpoint, you can obtain search results based on specified query and optionally filter results by -{fields_to_md(AudioSearchQueryStringSerializer.fields_names)}. +{fields_to_md(AudioSearchRequestSerializer.fields_names)}. {MediaSearch.desc}""" # noqa @@ -50,7 +50,7 @@ class AudioSearch(MediaSearch): "200": openapi.Response( description="OK", examples=audio_search_200_example, - schema=AudioSearchResultsSerializer(many=True) + schema=AudioSearchSerializer(many=True) ), "400": openapi.Response( description="Bad Request", @@ -69,7 +69,7 @@ class AudioSearch(MediaSearch): swagger_setup = { 'operation_id': 'audio_search', 'operation_description': desc, - 'query_serializer': AudioSearchQueryStringSerializer, + 'query_serializer': AudioSearchRequestSerializer, 'responses': responses, 'code_examples': code_examples } @@ -194,7 +194,7 @@ class AudioComplain(MediaComplain): "201": openapi.Response( description="OK", examples=audio_report_create_201_example, - schema=ReportAudioSerializer + schema=AudioReportSerializer ) } @@ -208,7 +208,7 @@ class AudioComplain(MediaComplain): swagger_setup = { 'operation_id': 'audio_report', 'operation_description': desc, - 'query_serializer': ReportAudioSerializer, + 'query_serializer': AudioReportSerializer, 'responses': responses, 'code_examples': code_examples, } diff --git a/openverse-api/catalog/api/docs/image_docs.py b/openverse-api/catalog/api/docs/image_docs.py index 12903c4b3..2c2570dd2 100644 --- a/openverse-api/catalog/api/docs/image_docs.py +++ b/openverse-api/catalog/api/docs/image_docs.py @@ -32,10 +32,10 @@ NotFoundErrorSerializer, ) from catalog.api.serializers.image_serializers import ( - ImageSearchQueryStringSerializer, - ImageSearchResultsSerializer, + ImageSearchRequestSerializer, + ImageSearchSerializer, ImageSerializer, - ReportImageSerializer, + ImageReportSerializer, OembedRequestSerializer, OembedSerializer, ) @@ -48,7 +48,7 @@ class ImageSearch(MediaSearch): By using this endpoint, you can obtain search results based on specified query and optionally filter results by -{fields_to_md(ImageSearchQueryStringSerializer.fields_names)}. +{fields_to_md(ImageSearchRequestSerializer.fields_names)}. {MediaSearch.desc}""" # noqa @@ -56,7 +56,7 @@ class ImageSearch(MediaSearch): "200": openapi.Response( description="OK", examples=image_search_200_example, - schema=ImageSearchResultsSerializer + schema=ImageSearchSerializer ), "400": openapi.Response( description="Bad Request", @@ -75,7 +75,7 @@ class ImageSearch(MediaSearch): swagger_setup = { 'operation_id': 'image_search', 'operation_description': desc, - 'query_serializer': ImageSearchQueryStringSerializer, + 'query_serializer': ImageSearchRequestSerializer, 'responses': responses, 'code_examples': code_examples, } @@ -200,7 +200,7 @@ class ImageComplain(MediaComplain): "201": openapi.Response( description="OK", examples=images_report_create_201_example, - schema=ReportImageSerializer + schema=ImageReportSerializer ) } @@ -214,7 +214,7 @@ class ImageComplain(MediaComplain): swagger_setup = { 'operation_id': 'image_report', 'operation_description': desc, - 'query_serializer': ReportImageSerializer, + 'query_serializer': ImageReportSerializer, 'responses': responses, 'code_examples': code_examples, } diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index cbde630d2..9ae849511 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -5,17 +5,17 @@ from catalog.api.models import AudioReport from catalog.api.serializers.media_serializers import ( _validate_enum, - MediaSearchQueryStringSerializer, - MediaSearchResultsSerializer, + MediaSearchRequestSerializer, + MediaSearchSerializer, MediaSerializer, ) -class AudioSearchQueryStringSerializer(MediaSearchQueryStringSerializer): +class AudioSearchRequestSerializer(MediaSearchRequestSerializer): """ Parse and validate search query string parameters. """ fields_names = [ - *MediaSearchQueryStringSerializer.fields_names, + *MediaSearchRequestSerializer.fields_names, 'source', 'categories', 'duration', @@ -160,7 +160,7 @@ def get_waveform(self, obj): return f'https://{host}{path}' -class AudioSearchResultsSerializer(MediaSearchResultsSerializer): +class AudioSearchSerializer(MediaSearchSerializer): """ The full audio search response. This serializer is purely representational and not actually used to @@ -176,11 +176,11 @@ class AudioSearchResultsSerializer(MediaSearchResultsSerializer): ) -class ReportAudioSerializer(serializers.ModelSerializer): +class AudioReportSerializer(serializers.ModelSerializer): class Meta: model = AudioReport fields = ('id', 'identifier', 'reason', 'description') - read_only_fields = ('id', 'identifier',) + read_only_fields = ('id', 'identifier') def create(self, validated_data): if validated_data['reason'] == "other" and \ diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index 929ce21da..94a305a36 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -6,17 +6,17 @@ from catalog.api.serializers.media_serializers import ( _add_protocol, _validate_enum, - MediaSearchQueryStringSerializer, - MediaSearchResultsSerializer, + MediaSearchRequestSerializer, + MediaSearchSerializer, MediaSerializer, ) -class ImageSearchQueryStringSerializer(MediaSearchQueryStringSerializer): +class ImageSearchRequestSerializer(MediaSearchRequestSerializer): """ Parse and validate search query string parameters. """ fields_names = [ - *MediaSearchQueryStringSerializer.fields_names, + *MediaSearchRequestSerializer.fields_names, 'source', 'categories', 'aspect_ratio', @@ -129,7 +129,7 @@ def get_thumbnail(self, obj): return f'https://{host}{path}' -class ImageSearchResultsSerializer(MediaSearchResultsSerializer): +class ImageSearchSerializer(MediaSearchSerializer): """ The full image search response. This serializer is purely representational and not actually used to @@ -157,6 +157,22 @@ def validate_url(value): return _add_protocol(value) +class ImageReportSerializer(serializers.ModelSerializer): + class Meta: + model = ImageReport + fields = ('id', 'identifier', 'reason', 'description') + read_only_fields = ('id', 'identifier') + + def create(self, validated_data): + if validated_data['reason'] == "other" and \ + ('description' not in validated_data or len( + validated_data['description'])) < 20: + raise serializers.ValidationError( + "Description must be at least be 20 characters long" + ) + return ImageReport.objects.create(**validated_data) + + class OembedSerializer(serializers.ModelSerializer): """ The embedded content from a specified image URL. """ version = serializers.ReadOnlyField( @@ -208,7 +224,7 @@ def get_height(self, obj): return self.context.get('height', obj.height) -class WatermarkQueryStringSerializer(serializers.Serializer): +class WatermarkRequestSerializer(serializers.Serializer): embed_metadata = serializers.BooleanField( help_text="Whether to embed ccREL metadata via XMP.", default=True @@ -218,19 +234,3 @@ class WatermarkQueryStringSerializer(serializers.Serializer): " text at the bottom.", default=True ) - - -class ReportImageSerializer(serializers.ModelSerializer): - class Meta: - model = ImageReport - fields = ('id', 'identifier', 'reason', 'description') - read_only_fields = ('id', 'identifier',) - - def create(self, validated_data): - if validated_data['reason'] == "other" and \ - ('description' not in validated_data or len( - validated_data['description'])) < 20: - raise serializers.ValidationError( - "Description must be at least be 20 characters long" - ) - return ImageReport.objects.create(**validated_data) diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index 4f6939636..d5aaa9d89 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -81,7 +81,7 @@ class TagSerializer(serializers.Serializer): ) -class MediaSearchQueryStringSerializer(serializers.Serializer): +class MediaSearchRequestSerializer(serializers.Serializer): """ This serializer parses and validates search query string parameters. """ @@ -341,7 +341,7 @@ def validate_foreign_landing_url(self, value): return _add_protocol(value) -class MediaSearchResultsSerializer(serializers.Serializer): +class MediaSearchSerializer(serializers.Serializer): """ This serializer serializes the full media search response. The class should be inherited by all individual media serializers. diff --git a/openverse-api/catalog/api/views/audio_views.py b/openverse-api/catalog/api/views/audio_views.py index 36ae1ef0c..df5fa66c6 100644 --- a/openverse-api/catalog/api/views/audio_views.py +++ b/openverse-api/catalog/api/views/audio_views.py @@ -14,9 +14,9 @@ ) from catalog.api.models import Audio from catalog.api.serializers.audio_serializers import ( - AudioSearchQueryStringSerializer, + AudioSearchRequestSerializer, AudioSerializer, - ReportAudioSerializer, + AudioReportSerializer, AudioWaveformSerializer, ) from catalog.api.utils.exceptions import get_api_exception @@ -45,7 +45,7 @@ class AudioViewSet(MediaViewSet): """ model_class = Audio - query_serializer_class = AudioSearchQueryStringSerializer + query_serializer_class = AudioSearchRequestSerializer default_index = 'audio' qa_index = 'search-qa-audio' @@ -98,6 +98,6 @@ def waveform(self, *_, **__): @action(detail=True, methods=['post'], - serializer_class=ReportAudioSerializer) + serializer_class=AudioReportSerializer) def report(self, *args, **kwargs): return self._report(*args, **kwargs) diff --git a/openverse-api/catalog/api/views/image_views.py b/openverse-api/catalog/api/views/image_views.py index 5977e2eae..5fac1db4e 100644 --- a/openverse-api/catalog/api/views/image_views.py +++ b/openverse-api/catalog/api/views/image_views.py @@ -22,10 +22,10 @@ ) from catalog.api.models import Image from catalog.api.serializers.image_serializers import ( - ImageSearchQueryStringSerializer, + ImageSearchRequestSerializer, ImageSerializer, - ReportImageSerializer, - WatermarkQueryStringSerializer, + ImageReportSerializer, + WatermarkRequestSerializer, OembedRequestSerializer, OembedSerializer, ) @@ -52,7 +52,7 @@ class ImageViewSet(MediaViewSet): """ model_class = Image - query_serializer_class = ImageSearchQueryStringSerializer + query_serializer_class = ImageSearchRequestSerializer default_index = 'image' qa_index = 'search-qa-image' @@ -111,7 +111,7 @@ def watermark(self, request, *_, **__): if not settings.WATERMARK_ENABLED: raise Http404 # watermark feature is disabled - params = WatermarkQueryStringSerializer(data=request.query_params) + params = WatermarkRequestSerializer(data=request.query_params) params.is_valid(raise_exception=True) image = self.get_object() @@ -159,7 +159,7 @@ def watermark(self, request, *_, **__): @action(detail=True, methods=['post'], - serializer_class=ReportImageSerializer) + serializer_class=ImageReportSerializer) def report(self, *args, **kwargs): return super().report(*args, **kwargs) From 136af051dd97ba9c39b6305d6753de214358c4a1 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 01:53:27 +0530 Subject: [PATCH 20/37] Programmatically generate list of fields --- .../catalog/api/serializers/audio_serializers.py | 10 +++++----- .../catalog/api/serializers/image_serializers.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index 9ae849511..1e624a4bb 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -2,6 +2,7 @@ from rest_framework import serializers from catalog.api.controllers.search_controller import get_sources +from catalog.api.docs.media_docs import fields_to_md from catalog.api.models import AudioReport from catalog.api.serializers.media_serializers import ( _validate_enum, @@ -168,11 +169,10 @@ class AudioSearchSerializer(MediaSearchSerializer): """ results = AudioSerializer( many=True, - help_text="An array of audios and their details such as `title`, `id`, " - "`creator`, `creator_url`, `url`, `provider`, `source`, " - "`license`, `license_version`, `license_url`, " - "`foreign_landing_url`, `detail_url`, `related_url`, " - "and `fields_matched `." + help_text=( + "An array of audios and their details such as " + f"{fields_to_md(AudioSerializer.fields_names)}." + ), ) diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index 94a305a36..933532939 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -2,6 +2,7 @@ from rest_framework import serializers from catalog.api.controllers.search_controller import get_sources +from catalog.api.docs.media_docs import fields_to_md from catalog.api.models import Image, ImageReport from catalog.api.serializers.media_serializers import ( _add_protocol, @@ -137,11 +138,10 @@ class ImageSearchSerializer(MediaSearchSerializer): """ results = ImageSerializer( many=True, - help_text="An array of images and their details such as `title`, `id`, " - "`creator`, `creator_url`, `url`, `thumbnail`, `provider`, " - "`source`, `license`, `license_version`, `license_url`, " - "`foreign_landing_url`, `detail_url`, `related_url`, " - "and `fields_matched `." + help_text=( + "An array of images and their details such as " + f"{fields_to_md(ImageSerializer.fields_names)}." + ), ) From d0c8b33581dac379309e106b097a35e6a526d680 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 01:53:35 +0530 Subject: [PATCH 21/37] Remove unused serializer --- .../catalog/api/serializers/media_serializers.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index d5aaa9d89..c12ece255 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -359,15 +359,3 @@ class MediaSearchSerializer(serializers.Serializer): page = serializers.IntegerField( help_text="The current page number returned in the response." ) - - -class ProxiedImageSerializer(serializers.Serializer): - """ - We want to show 3rd party content securely and under our own native URLs, so - we route some images through our own proxy. We use this same endpoint to - generate thumbnails for content. - """ - full_size = serializers.BooleanField( - default=False, - help_text="If set, do not thumbnail the image." - ) From bfb25a0624460c3867453689e064177465cb4810 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 02:57:14 +0530 Subject: [PATCH 22/37] Use cleaner validation for deprecated fields, making post-error redundant --- .../api/serializers/media_serializers.py | 32 ++++---- openverse-api/catalog/api/utils/exceptions.py | 74 ++++++------------- openverse-api/catalog/settings.py | 1 + 3 files changed, 39 insertions(+), 68 deletions(-) diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index c12ece255..003b94e29 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -174,45 +174,47 @@ class MediaSearchRequestSerializer(serializers.Serializer): ) @staticmethod - def validate_q(value): - if len(value) > 200: - return value[0:199] - else: - return value + def _truncate(value): + max_length = 200 + return value if len(value) <= max_length else value[:max_length] + + def validate_q(self, value): + return self._truncate(value) @staticmethod def validate_license(value): + """Checks whether license is a valid license code.""" return _validate_li(value) @staticmethod def validate_license_type(value): - """ - Resolves a list of license types to a list of licenses. - Example: commercial -> ['BY', 'BY-SA', 'BY-ND', 'CC0', 'PDM'] - """ + """Checks whether license type is a known collection of licenses.""" return _validate_lt(value) def validate_creator(self, value): - return self.validate_q(value) + return self._truncate(value) def validate_tags(self, value): - return self.validate_q(value) + return self._truncate(value) def validate_title(self, value): - return self.validate_q(value) + return self._truncate(value) @staticmethod def validate_extension(value): return value.lower() def validate(self, data): + errors = {} for deprecated in self.deprecated_params: param, successor = deprecated if param in self.initial_data: - raise serializers.ValidationError( - f"Parameter '{param}' is deprecated in this release of" - f" the API. Use '{successor}' instead." + errors[param] = ( + f"Parameter '{param}' is deprecated in this release of the " + f"API. Use '{successor}' instead." ) + if errors: + raise serializers.ValidationError(errors) return data diff --git a/openverse-api/catalog/api/utils/exceptions.py b/openverse-api/catalog/api/utils/exceptions.py index ecb39b15c..137a1bc1c 100644 --- a/openverse-api/catalog/api/utils/exceptions.py +++ b/openverse-api/catalog/api/utils/exceptions.py @@ -1,57 +1,6 @@ -from rest_framework import status +from rest_framework.serializers import ValidationError from rest_framework.exceptions import APIException -from rest_framework.response import Response -""" -Override the presentation of ValidationErrors, which are deeply nested and -difficult to parse. - -Note that error 500 pages are not handled here; they are generated by the -production web server configuration, and not reproducible locally. -""" - - -def parse_value_errors(errors): - fields = ['q'] - messages = [errors.args[0].info['error']['root_cause'][0]['reason']] - return fields, messages - - -def parse_non_value_errors(errors): - fields = [f for f in errors] - messages = [] - for _field in errors: - error = errors[_field] - for e in error: - messages.append(e) - - # Don't return "non field errors" in deprecation exceptions. There is no - # other way to recover the affected fields other than parsing the error. - if fields == ['non_field_errors']: - split_error = list(messages) - field_idx = ' '.join(messages).index('Parameter') + 1 - fields = [split_error[field_idx].replace("'", '')][0] - - return fields, messages - - -def input_error_response(errors): - if isinstance(errors, ValueError): - fields, messages = parse_value_errors(errors) - else: - fields, messages = parse_non_value_errors(errors) - - detail = "Invalid input given for fields." - for i, _ in enumerate(fields): - detail += f" '{fields[i]}' -> {messages[i]}" - - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - 'error': 'InputError', - 'detail': detail, - 'fields': fields - } - ) +from rest_framework.views import exception_handler as drf_exception_handler def get_api_exception(error_message, response_code=500, error_code=None): @@ -72,3 +21,22 @@ class SubAPIException(APIException): default_detail = error_message default_code = error_code return SubAPIException() + + +def exception_handler(ex, context): + """ + Handle the exception raised in a DRF context. See `DRF docs`_. + .. _DRF docs: https://www.django-rest-framework.org/api-guide/exceptions/#custom-exception-handling # noqa: E501 + + :param ex: the exception that has occurred + :param context: additional data about the context of the exception + :return: the response to show for the exception + """ + + res = drf_exception_handler(ex, context) + if isinstance(ex, ValidationError): + # Wrap validation errors inside a `detail` key for consistency + res.data = { + 'detail': res.data + } + return res diff --git a/openverse-api/catalog/settings.py b/openverse-api/catalog/settings.py index d6d54d111..0d0d550f8 100644 --- a/openverse-api/catalog/settings.py +++ b/openverse-api/catalog/settings.py @@ -125,6 +125,7 @@ 'enhanced_oauth2_client_credentials_sustained': '20000/day', 'enhanced_oauth2_client_credentials_burst': '200/min' }, + 'EXCEPTION_HANDLER': 'catalog.api.utils.exceptions.exception_handler', } if os.environ.get('DISABLE_GLOBAL_THROTTLING', default=False) in true_strings: From b828ed6496d0731bd86af916d32a5f63114d2a8b Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 02:57:24 +0530 Subject: [PATCH 23/37] Update error serializers --- .../api/serializers/error_serializers.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/openverse-api/catalog/api/serializers/error_serializers.py b/openverse-api/catalog/api/serializers/error_serializers.py index 12e814a21..a0fd87470 100644 --- a/openverse-api/catalog/api/serializers/error_serializers.py +++ b/openverse-api/catalog/api/serializers/error_serializers.py @@ -1,37 +1,38 @@ +""" +Reference for exception handling in Django REST Framework: + https://www.django-rest-framework.org/api-guide/exceptions/ +""" + from rest_framework import serializers class InputErrorSerializer(serializers.Serializer): """ Returned if invalid query parameters are passed. """ - error = serializers.CharField( - help_text="The name of error." - ) - detail = serializers.CharField( - help_text="The description for error." - ) - fields = serializers.ListField( - help_text="List of query parameters that causes error." + + detail = serializers.DictField( + help_text="Mapping of field names with errors from to that field.", + child=serializers.ListField( + child=serializers.CharField() + ) ) class NotFoundErrorSerializer(serializers.Serializer): """ Returned if the requested content could not be found. """ detail = serializers.CharField( - help_text="The description for error" + help_text="The description for error." ) class ForbiddenErrorSerializer(serializers.Serializer): - """ Returned if access to requested content is forbidden for - some reason.""" + """ Returned if access to requested content is forbidden. """ detail = serializers.CharField( - help_text="The description for error" + help_text="The description for error." ) class InternalServerErrorSerializer(serializers.Serializer): - """ Returned if the request could not be processed by the server for an - unknown reason.""" + """ Returned if the request could not be processed. """ detail = serializers.CharField( - help_text="The description for error" + help_text="The description for error." ) From c19a0d9b2cf7be62fb4a17688e8da37772da2794 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 12:38:21 +0530 Subject: [PATCH 24/37] Fix broken reference in audio report endpoint --- openverse-api/catalog/api/views/audio_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openverse-api/catalog/api/views/audio_views.py b/openverse-api/catalog/api/views/audio_views.py index df5fa66c6..5efc2702a 100644 --- a/openverse-api/catalog/api/views/audio_views.py +++ b/openverse-api/catalog/api/views/audio_views.py @@ -100,4 +100,4 @@ def waveform(self, *_, **__): methods=['post'], serializer_class=AudioReportSerializer) def report(self, *args, **kwargs): - return self._report(*args, **kwargs) + return super().report(*args, **kwargs) From ceafca2ae654d3a91606875a347b4f9979a8d32c Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 12:59:10 +0530 Subject: [PATCH 25/37] Remove field 'id' --- openverse-api/catalog/api/serializers/audio_serializers.py | 4 ++-- openverse-api/catalog/api/serializers/image_serializers.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index 1e624a4bb..453d4ed47 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -179,8 +179,8 @@ class AudioSearchSerializer(MediaSearchSerializer): class AudioReportSerializer(serializers.ModelSerializer): class Meta: model = AudioReport - fields = ('id', 'identifier', 'reason', 'description') - read_only_fields = ('id', 'identifier') + fields = ('identifier', 'reason', 'description') + read_only_fields = ('identifier',) def create(self, validated_data): if validated_data['reason'] == "other" and \ diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index 933532939..e618c269b 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -160,8 +160,8 @@ def validate_url(value): class ImageReportSerializer(serializers.ModelSerializer): class Meta: model = ImageReport - fields = ('id', 'identifier', 'reason', 'description') - read_only_fields = ('id', 'identifier') + fields = ('identifier', 'reason', 'description') + read_only_fields = ('identifier',) def create(self, validated_data): if validated_data['reason'] == "other" and \ From 9d120d94bcf93fc7317af9b54dcc4c8841d8b5e6 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 15:21:47 +0530 Subject: [PATCH 26/37] Use `HyperlinkedIdentityField` to automatically generate related URLs --- .../api/serializers/audio_serializers.py | 33 ++++++++----------- .../api/serializers/image_serializers.py | 15 ++++----- 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index 453d4ed47..ee96f49ab 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -1,4 +1,3 @@ -from django.urls import reverse from rest_framework import serializers from catalog.api.controllers.search_controller import get_sources @@ -96,12 +95,6 @@ class AudioSerializer(MediaSerializer): used to generate Swagger documentation. """ - thumbnail = serializers.SerializerMethodField( - help_text="A direct link to the miniature artwork." - ) - waveform = serializers.SerializerMethodField( - help_text='A direct link to the waveform peaks.' - ) audio_set = serializers.PrimaryKeyRelatedField( required=False, help_text='Reference to set of which this track is a part.', @@ -135,6 +128,18 @@ class AudioSerializer(MediaSerializer): ) # Hyperlinks + thumbnail = serializers.HyperlinkedIdentityField( + read_only=True, + view_name='audio-thumb', + lookup_field='identifier', + help_text="A direct link to the miniature artwork." + ) + waveform = serializers.HyperlinkedIdentityField( + read_only=True, + view_name='audio-waveform', + lookup_field='identifier', + help_text='A direct link to the waveform peaks.' + ) detail_url = serializers.HyperlinkedIdentityField( read_only=True, view_name='audio-detail', @@ -142,24 +147,12 @@ class AudioSerializer(MediaSerializer): help_text="A direct link to the detail view of this audio file." ) related_url = serializers.HyperlinkedIdentityField( + read_only=True, view_name='audio-related', lookup_field='identifier', - read_only=True, help_text="A link to an endpoint that provides similar audio files." ) - def get_thumbnail(self, obj): - request = self.context['request'] - host = request.get_host() - path = reverse('audio-thumb', kwargs={'identifier': obj.identifier}) - return f'https://{host}{path}' - - def get_waveform(self, obj): - request = self.context['request'] - host = request.get_host() - path = reverse('audio-waveform', kwargs={'identifier': obj.identifier}) - return f'https://{host}{path}' - class AudioSearchSerializer(MediaSearchSerializer): """ diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index e618c269b..f18927c0a 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -97,9 +97,6 @@ class ImageSerializer(MediaSerializer): used to generate Swagger documentation. """ - thumbnail = serializers.SerializerMethodField( - help_text="A direct link to the miniature image." - ) height = serializers.IntegerField( required=False, help_text="The height of the image in pixels. Not always available." @@ -110,6 +107,12 @@ class ImageSerializer(MediaSerializer): ) # Hyperlinks + thumbnail = serializers.HyperlinkedIdentityField( + read_only=True, + view_name='image-thumb', + lookup_field='identifier', + help_text="A direct link to the miniature image." + ) detail_url = serializers.HyperlinkedIdentityField( read_only=True, view_name='image-detail', @@ -123,12 +126,6 @@ class ImageSerializer(MediaSerializer): help_text="A link to an endpoint that provides similar images." ) - def get_thumbnail(self, obj): - request = self.context['request'] - host = request.get_host() - path = reverse('image-thumb', kwargs={'identifier': obj.identifier}) - return f'https://{host}{path}' - class ImageSearchSerializer(MediaSearchSerializer): """ From 4862f95141e8e1f923f335a31bf4b0ca2ba07c79 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 16:34:03 +0530 Subject: [PATCH 27/37] Fix nomenclature in example requests and responses --- openverse-api/catalog/api/docs/audio_docs.py | 28 +-- openverse-api/catalog/api/docs/image_docs.py | 40 ++-- .../catalog/api/examples/__init__.py | 36 ++-- .../catalog/api/examples/audio_requests.py | 58 +++--- .../catalog/api/examples/audio_responses.py | 139 +++++++++----- .../catalog/api/examples/image_requests.py | 64 ++++--- .../catalog/api/examples/image_responses.py | 178 ++++++++++-------- 7 files changed, 319 insertions(+), 224 deletions(-) diff --git a/openverse-api/catalog/api/docs/audio_docs.py b/openverse-api/catalog/api/docs/audio_docs.py index 785004007..580293cbd 100644 --- a/openverse-api/catalog/api/docs/audio_docs.py +++ b/openverse-api/catalog/api/docs/audio_docs.py @@ -9,19 +9,19 @@ MediaComplain, ) from catalog.api.examples import ( - audio_search_curl, + audio_search_list_curl, audio_search_200_example, audio_search_400_example, - recommendations_audio_read_curl, - recommendations_audio_read_200_example, - recommendations_audio_read_404_example, + audio_stats_curl, + audio_stats_200_example, audio_detail_curl, audio_detail_200_example, audio_detail_404_example, - audio_stats_curl, - audio_stats_200_example, - report_audio_curl, - audio_report_create_201_example, + audio_related_curl, + audio_related_200_example, + audio_related_404_example, + audio_complain_curl, + audio_complain_201_example, ) from catalog.api.serializers.audio_serializers import ( AudioSearchRequestSerializer, @@ -62,7 +62,7 @@ class AudioSearch(MediaSearch): code_examples = [ { 'lang': 'Bash', - 'source': audio_search_curl, + 'source': audio_search_list_curl, }, ] @@ -155,12 +155,12 @@ class AudioRelated(MediaRelated): responses = { "200": openapi.Response( description="OK", - examples=recommendations_audio_read_200_example, + examples=audio_related_200_example, schema=AudioSerializer ), "404": openapi.Response( description="Not Found", - examples=recommendations_audio_read_404_example, + examples=audio_related_404_example, schema=NotFoundErrorSerializer ) } @@ -168,7 +168,7 @@ class AudioRelated(MediaRelated): code_examples = [ { 'lang': 'Bash', - 'source': recommendations_audio_read_curl, + 'source': audio_related_curl, }, ] @@ -193,7 +193,7 @@ class AudioComplain(MediaComplain): responses = { "201": openapi.Response( description="OK", - examples=audio_report_create_201_example, + examples=audio_complain_201_example, schema=AudioReportSerializer ) } @@ -201,7 +201,7 @@ class AudioComplain(MediaComplain): code_examples = [ { 'lang': 'Bash', - 'source': report_audio_curl, + 'source': audio_complain_curl, } ] diff --git a/openverse-api/catalog/api/docs/image_docs.py b/openverse-api/catalog/api/docs/image_docs.py index 2c2570dd2..58ade5fca 100644 --- a/openverse-api/catalog/api/docs/image_docs.py +++ b/openverse-api/catalog/api/docs/image_docs.py @@ -10,22 +10,22 @@ MediaComplain, ) from catalog.api.examples import ( - image_search_curl, + image_search_list_curl, image_search_200_example, image_search_400_example, - recommendations_images_read_curl, - recommendations_images_read_200_example, - recommendations_images_read_404_example, + image_stats_curl, + image_stats_200_example, image_detail_curl, image_detail_200_example, image_detail_404_example, - image_stats_curl, - image_stats_200_example, - report_image_curl, - images_report_create_201_example, - oembed_list_curl, - oembed_list_200_example, - oembed_list_404_example, + image_related_curl, + image_related_200_example, + image_related_404_example, + image_complain_curl, + image_complain_201_example, + image_oembed_curl, + image_oembed_200_example, + image_oembed_404_example, ) from catalog.api.serializers.error_serializers import ( InputErrorSerializer, @@ -68,7 +68,7 @@ class ImageSearch(MediaSearch): code_examples = [ { 'lang': 'Bash', - 'source': image_search_curl, + 'source': image_search_list_curl, }, ] @@ -161,12 +161,12 @@ class ImageRelated(MediaRelated): responses = { "200": openapi.Response( description="OK", - examples=recommendations_images_read_200_example, + examples=image_related_200_example, schema=ImageSerializer ), "404": openapi.Response( description="Not Found", - examples=recommendations_images_read_404_example, + examples=image_related_404_example, schema=NotFoundErrorSerializer ) } @@ -174,7 +174,7 @@ class ImageRelated(MediaRelated): code_examples = [ { 'lang': 'Bash', - 'source': recommendations_images_read_curl + 'source': image_related_curl } ] @@ -199,7 +199,7 @@ class ImageComplain(MediaComplain): responses = { "201": openapi.Response( description="OK", - examples=images_report_create_201_example, + examples=image_complain_201_example, schema=ImageReportSerializer ) } @@ -207,7 +207,7 @@ class ImageComplain(MediaComplain): code_examples = [ { 'lang': 'Bash', - 'source': report_image_curl, + 'source': image_complain_curl, } ] @@ -234,12 +234,12 @@ class ImageOembed: responses = { "200": openapi.Response( description="OK", - examples=oembed_list_200_example, + examples=image_oembed_200_example, schema=OembedSerializer ), "404": openapi.Response( description="Not Found", - examples=oembed_list_404_example, + examples=image_oembed_404_example, schema=NotFoundErrorSerializer ) } @@ -247,7 +247,7 @@ class ImageOembed: code_examples = [ { 'lang': 'Bash', - 'source': oembed_list_curl, + 'source': image_oembed_curl, }, ] diff --git a/openverse-api/catalog/api/examples/__init__.py b/openverse-api/catalog/api/examples/__init__.py index e2117ee5d..550027607 100644 --- a/openverse-api/catalog/api/examples/__init__.py +++ b/openverse-api/catalog/api/examples/__init__.py @@ -1,37 +1,39 @@ from catalog.api.examples.audio_requests import ( + audio_search_list_curl, audio_search_curl, - recommendations_audio_read_curl, - audio_detail_curl, audio_stats_curl, - report_audio_curl, + audio_detail_curl, + audio_related_curl, + audio_complain_curl, ) from catalog.api.examples.audio_responses import ( audio_search_200_example, audio_search_400_example, + audio_stats_200_example, audio_detail_200_example, audio_detail_404_example, - recommendations_audio_read_200_example, - recommendations_audio_read_404_example, - audio_report_create_201_example, - audio_stats_200_example, + audio_related_200_example, + audio_related_404_example, + audio_complain_201_example, ) from catalog.api.examples.image_requests import ( + image_search_list_curl, image_search_curl, - recommendations_images_read_curl, - image_detail_curl, image_stats_curl, - report_image_curl, - oembed_list_curl, + image_detail_curl, + image_related_curl, + image_complain_curl, + image_oembed_curl, ) from catalog.api.examples.image_responses import ( image_search_200_example, image_search_400_example, + image_stats_200_example, image_detail_200_example, image_detail_404_example, - recommendations_images_read_200_example, - recommendations_images_read_404_example, - oembed_list_200_example, - oembed_list_404_example, - images_report_create_201_example, - image_stats_200_example, + image_related_200_example, + image_related_404_example, + image_complain_201_example, + image_oembed_200_example, + image_oembed_404_example, ) diff --git a/openverse-api/catalog/api/examples/audio_requests.py b/openverse-api/catalog/api/examples/audio_requests.py index 9074556e3..fc2e7ab5a 100644 --- a/openverse-api/catalog/api/examples/audio_requests.py +++ b/openverse-api/catalog/api/examples/audio_requests.py @@ -1,3 +1,11 @@ +import os + +token = os.getenv('AUDIO_REQ_TOKEN', 'DLBYIcfnKfolaXKcmMC8RIDCavc2hW') +origin = os.getenv('AUDIO_REQ_ORIGIN', 'https://api.openverse.engineering') + +auth = f'-H "Authorization: Bearer {token}"' if token else '' +identifier = '440a0240-8b20-49e2-a4e6-6fee550fcc41' + syntax_examples = { "using single query parameter": 'test', @@ -20,29 +28,37 @@ 'theatre~1', } -audio_search_curl = '\n\n'.join([ - (f'# Example {index}: Search for audio {purpose}\n' - 'curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" ' - f'https://api.openverse.engineering/v1/audio?q={syntax}') - for (index, (purpose, syntax)) in enumerate(syntax_examples.items()) -]) +audio_search_list_curl = '\n'.join(f""" +# Example {index}: Search for audio {purpose} +curl {auth} "{origin}/v1/audio/?q={syntax}" +""" for (index, (purpose, syntax)) in enumerate(syntax_examples.items())) -recommendations_audio_read_curl = """ -# Get related audio files for audio ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/recommendations/audio/7c829a03-fb24-4b57-9b03-65f43ed19395 -""" # noqa +audio_search_curl = f""" +# Search for music titled "Friend" by Rob Costlow +curl {auth} "{origin}/v1/audio/?title=Friend&creator=Rob%20Costlow" +""" -audio_detail_curl = """ -# Get the details of audio ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/audio/7c829a03-fb24-4b57-9b03-65f43ed19395 -""" # noqa - -audio_stats_curl = """ +audio_stats_curl = f""" # Get the statistics for audio sources -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/audio/stats -""" # noqa +curl {auth} "{origin}/v1/audio/stats/" +""" + +audio_detail_curl = f""" +# Get the details of audio ID {identifier} +curl {auth} "{origin}/v1/audio/{identifier}/" +""" + +audio_related_curl = f""" +# Get related audio files for audio ID {identifier} +curl {auth} "{origin}/v1/audio/{identifier}/related/" +""" -report_audio_curl = """ -# Report an issue about audio ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "description": "This audio contains sensitive content"}' https://api.openverse.engineering/v1/audio/7c829a03-fb24-4b57-9b03-65f43ed19395/report +audio_complain_curl = f""" +# Report an issue about audio ID {identifier} +curl \\ + -X POST \\ + -H "Content-Type: application/json" \\ + {auth} \\ + -d '{{"reason": "mature", "description": "This audio contains sensitive content"}}' \\ + "{origin}/v1/audio/{identifier}/report/" """ # noqa diff --git a/openverse-api/catalog/api/examples/audio_responses.py b/openverse-api/catalog/api/examples/audio_responses.py index 1bb686ad4..2d7a96965 100644 --- a/openverse-api/catalog/api/examples/audio_responses.py +++ b/openverse-api/catalog/api/examples/audio_responses.py @@ -1,28 +1,68 @@ +import os + +origin = os.getenv('AUDIO_REQ_ORIGIN', 'https://api.openverse.engineering') + +identifier = '440a0240-8b20-49e2-a4e6-6fee550fcc41' + +base_audio = { + "id": identifier, + "title": "Friend", + "foreign_landing_url": "https://www.jamendo.com/track/5786", + "creator": "Rob Costlow", + "creator_url": "https://www.jamendo.com/artist/125/rob.costlow", + "url": "https://mp3d.jamendo.com/download/track/5786/mp32", + "license": "by-nc-nd", + "license_version": "3.0", + "license_url": "https://creativecommons.org/licenses/by-nc-nd/3.0/", + "provider": "jamendo", + "source": "jamendo", + "tags": [ + { + "name": "instrumental" + }, + { + "name": "neutral" + }, + { + "name": "speed_medium" + }, + { + "name": "piano" + }, + { + "name": "strings" + }, + { + "name": "love" + }, + { + "name": "upbeat" + }, + { + "name": "neutral" + } + ], + "genres": [ + "newage" + ], + "thumbnail": f"{origin}/v1/audio/{identifier}/thumb/", + "waveform": f"{origin}/v1/audio/{identifier}/waveform/", + "detail_url": f"{origin}/v1/audio/{identifier}/", + "related_url": f"{origin}/v1/audio/{identifier}/related/" +} + audio_search_200_example = { "application/json": { - "result_count": 77, - "page_count": 77, - "page_size": 1, + "result_count": 1, + "page_count": 0, + "page_size": 20, + "page": 1, "results": [ - { - "title": "File:Mozart - Eine kleine Nachtmusik - 1. Allegro.ogg", # noqa - "id": "36537842-b067-4ca0-ad67-e00ff2e06b2e", - "creator": "Wolfgang Amadeus Mozart", - "creator_url": "https://en.wikipedia.org/wiki/Wolfgang_Amadeus_Mozart", # noqa - "url": "https://upload.wikimedia.org/wikipedia/commons/2/24/Mozart_-_Eine_kleine_Nachtmusik_-_1._Allegro.ogg", # noqa - "provider": "wikimedia", - "source": "wikimedia", - "license": "by-sa", - "license_version": "2.0", - "license_url": "https://creativecommons.org/licenses/by-sa/2.0/", # noqa - "foreign_landing_url": "https://commons.wikimedia.org/w/index.php?curid=3536953", # noqa - "detail_url": "http://api.openverse.engineering/v1/audio/36537842-b067-4ca0-ad67-e00ff2e06b2e", # noqa - "related_url": "http://api.openverse.engineering/v1/recommendations/audio/36537842-b067-4ca0-ad67-e00ff2e06b2e", # noqa + base_audio | { "fields_matched": [ - "description", "title" ] - } + }, ] }, } @@ -37,7 +77,36 @@ } } -recommendations_audio_read_200_example = { +audio_stats_200_example = { + "application/json": [ + { + "source_name": "jamendo", + "display_name": "Jamendo", + "source_url": "https://www.jamendo.com", + "logo_url": None, + "media_count": 5153, + } + ] +} + +audio_detail_200_example = { + "application/json": base_audio | { + "attribution": "\"Friend\" by Rob Costlow is licensed under CC-BY-NC-ND 3.0. To view a copy of this license, visit https://creativecommons.org/licenses/by-nc-nd/3.0/.", # noqa + "audio_set": None, + "duration": 240000, + "bit_rate": None, + "sample_rate": None, + "alt_files": None + }, +} + +audio_detail_404_example = { + "application/json": { + "detail": "Not found." + } +} + +audio_related_200_example = { "application/json": { "result_count": 10000, "page_count": 0, @@ -73,40 +142,16 @@ } } -recommendations_audio_read_404_example = { +audio_related_404_example = { "application/json": { "detail": "An internal server error occurred." } } -audio_detail_200_example = { +audio_complain_201_example = { "application/json": { - # TODO - } -} - -audio_detail_404_example = { - "application/json": { - "detail": "Not found." - } -} - -audio_report_create_201_example = { - "application/json": { - "id": 10, - "identifier": "7c829a03-fb24-4b57-9b03-65f43ed19395", + "identifier": identifier, "reason": "mature", "description": "This audio contains sensitive content" } } - -audio_stats_200_example = { - "application/json": [ - { - "source_name": "jamendo", - "audio_count": 123456789, - "display_name": "Jamendo", - "source_url": "https://www.jamendo.com" - } - ] -} diff --git a/openverse-api/catalog/api/examples/image_requests.py b/openverse-api/catalog/api/examples/image_requests.py index 906a82475..e0535b886 100644 --- a/openverse-api/catalog/api/examples/image_requests.py +++ b/openverse-api/catalog/api/examples/image_requests.py @@ -1,3 +1,11 @@ +import os + +token = os.getenv('AUDIO_REQ_TOKEN', 'DLBYIcfnKfolaXKcmMC8RIDCavc2hW') +origin = os.getenv('AUDIO_REQ_ORIGIN', 'https://api.openverse.engineering') + +auth = f'-H "Authorization: Bearer {token}"' if token else '' +identifier = '29cb352c-60c1-41d8-bfa1-7d6f7d955f63' + syntax_examples = { "using single query parameter": 'test', @@ -20,34 +28,42 @@ 'theatre~1', } -image_search_curl = '\n\n'.join([ - (f'# Example {index}: Search for images {purpose}\n' - 'curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" ' - f'https://api.openverse.engineering/v1/images?q={syntax}') - for (index, (purpose, syntax)) in enumerate(syntax_examples.items()) -]) +image_search_list_curl = '\n'.join(f""" +# Example {index}: Search for images {purpose} +curl {auth} "{origin}/v1/images?q={syntax}" +""" for (index, (purpose, syntax)) in enumerate(syntax_examples.items())) -recommendations_images_read_curl = """ -# Get related images for image ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/recommendations/images/7c829a03-fb24-4b57-9b03-65f43ed19395 -""" # noqa +image_search_curl = f""" +# Search for images titled "Bust" by Talbot +curl {auth} "{origin}/v1/images/?title=Bust&creator=Talbot" +""" -image_detail_curl = """ -# Get the details of image ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/images/7c829a03-fb24-4b57-9b03-65f43ed19395 -""" # noqa - -image_stats_curl = """ +image_stats_curl = f""" # Get the statistics for image sources -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" http://api.openverse.engineering/v1/images/stats -""" # noqa +curl {auth} "{origin}/v1/images/stats/" +""" + +image_detail_curl = f""" +# Get the details of image ID {identifier} +curl {auth} "{origin}/v1/images/{identifier}/" +""" + +image_related_curl = f""" +# Get related images for image ID {identifier} +curl {auth} "{origin}/v1/images/{identifier}/related/" +""" -report_image_curl = """ -# Report an issue about image ID 7c829a03-fb24-4b57-9b03-65f43ed19395 -curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" -d '{"reason": "mature", "description": "This image contains sensitive content"}' https://api.openverse.engineering/v1/images/7c829a03-fb24-4b57-9b03-65f43ed19395/report +image_complain_curl = f""" +# Report an issue about image ID {identifier} +curl \\ + -X POST \\ + -H "Content-Type: application/json" \\ + {auth} \\ + -d '{{"reason": "mature", "description": "This image contains sensitive content"}}' \\ + "{origin}/v1/images/{identifier}/report/" """ # noqa -oembed_list_curl = """ -# Retrieve embedded content from image URL (https://wordpress.org/openverse/photos/7c829a03-fb24-4b57-9b03-65f43ed19395) -curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" https://api.openverse.engineering/v1/oembed/?url=https://wordpress.org/openverse/photos/7c829a03-fb24-4b57-9b03-65f43ed19395 +image_oembed_curl = f""" +# Retrieve embedded content from image URL (https://wordpress.org/openverse/photos/{identifier}) +curl {auth} "{origin}/v1/images/oembed/?url=https://wordpress.org/openverse/photos/{identifier}" """ # noqa diff --git a/openverse-api/catalog/api/examples/image_responses.py b/openverse-api/catalog/api/examples/image_responses.py index ce4edcc19..1f3040aa6 100644 --- a/openverse-api/catalog/api/examples/image_responses.py +++ b/openverse-api/catalog/api/examples/image_responses.py @@ -1,28 +1,36 @@ +import os + +origin = os.getenv('AUDIO_REQ_ORIGIN', 'https://api.openverse.engineering') + +identifier = '29cb352c-60c1-41d8-bfa1-7d6f7d955f63' + +base_image = { + "id": identifier, + "title": "Bust of Patroclus (photograph; calotype; salt print)", + "foreign_landing_url": "https://collection.sciencemuseumgroup.org.uk/objects/co8554747/bust-of-patroclus-photograph-calotype-salt-print", # noqa + "creator": "William Henry Fox Talbot", + "url": "https://coimages.sciencemuseumgroup.org.uk/images/439/67/large_1937_1281_0001__0001_.jpg", # noqa + "license": "by-nc-nd", + "license_version": "4.0", + "license_url": "https://creativecommons.org/licenses/by-nc-nd/4.0/", + "provider": "sciencemuseum", + "source": "sciencemuseum", + "thumbnail": f"{origin}/v1/images/{identifier}/thumb/", + "detail_url": f"{origin}/v1/images/{identifier}/", + "related_url": f"{origin}/v1/images/{identifier}/related/" +} + image_search_200_example = { "application/json": { - "result_count": 77, - "page_count": 77, - "page_size": 1, + "result_count": 1, + "page_count": 0, + "page_size": 20, + "page": 1, "results": [ - { - "title": "File:Well test separator.svg", - "id": "36537842-b067-4ca0-ad67-e00ff2e06b2d", - "creator": "en:User:Oil&GasIndustry", - "creator_url": "https://en.wikipedia.org/wiki/User:Oil%26GasIndustry", # noqa - "url": "https://upload.wikimedia.org/wikipedia/commons/3/3a/Well_test_separator.svg", # noqa - "thumbnail": "https://api.openverse.engineering/v1/thumbs/36537842-b067-4ca0-ad67-e00ff2e06b2d", # noqa - "provider": "wikimedia", - "source": "wikimedia", - "license": "by", - "license_version": "3.0", - "license_url": "https://creativecommons.org/licenses/by/3.0", - "foreign_landing_url": "https://commons.wikimedia.org/w/index.php?curid=26229990", # noqa - "detail_url": "http://api.openverse.engineering/v1/images/36537842-b067-4ca0-ad67-e00ff2e06b2d", # noqa - "related_url": "http://api.openverse.engineering/v1/recommendations/images/36537842-b067-4ca0-ad67-e00ff2e06b2d", # noqa + base_image | { "fields_matched": [ - "description", "title" - ] + ], } ] }, @@ -38,7 +46,63 @@ } } -recommendations_images_read_200_example = { +image_stats_200_example = { + "application/json": [ + { + "source_name": "flickr", + "display_name": "Flickr", + "source_url": "https://www.flickr.com", + "logo_url": None, + "media_count": 1000 + }, + { + "source_name": "rawpixel", + "display_name": "rawpixel", + "source_url": "https://www.rawpixel.com", + "logo_url": None, + "media_count": 1000 + }, + { + "source_name": "sciencemuseum", + "display_name": "Science Museum", + "source_url": "https://www.sciencemuseum.org.uk", + "logo_url": None, + "media_count": 1000 + }, + { + "source_name": "stocksnap", + "display_name": "StockSnap", + "source_url": "https://stocksnap.io", + "logo_url": None, + "media_count": 1000 + }, + { + "source_name": "wikimedia", + "display_name": "Wikimedia", + "source_url": "https://commons.wikimedia.org", + "logo_url": None, + "media_count": 1000 + } + ] +} + +image_detail_200_example = { + "application/json": base_image | { + "attribution": "\"Bust of Patroclus (photograph; calotype; salt print)\" by William Henry Fox Talbot is licensed under CC-BY-NC-ND 4.0. To view a copy of this license, visit https://creativecommons.org/licenses/by-nc-nd/4.0/.", # noqa + "height": 1536, + "width": 1276, + "tags": None, + "creator_url": None, + } +} + +image_detail_404_example = { + "application/json": { + "detail": "Not found." + } +} + +image_related_200_example = { "application/json": { "result_count": 10000, "page_count": 0, @@ -71,83 +135,35 @@ } } -recommendations_images_read_404_example = { +image_related_404_example = { "application/json": { "detail": "An internal server error occurred." } } -image_detail_200_example = { - "application/json": { - "title": "exam test", - "id": "7c829a03-fb24-4b57-9b03-65f43ed19395", - "creator": "Sean MacEntee", - "creator_url": "https://www.flickr.com/photos/18090920@N07", - "tags": [ - { - "name": "exam" - }, - { - "name": "test" - } - ], - "url": "https://live.staticflickr.com/5122/5264886972_3234d62748.jpg", - "thumbnail": "https://api.openverse.engineering/v1/thumbs/7c829a03-fb24-4b57-9b03-65f43ed19395", # noqa - "provider": "flickr", - "source": "flickr", - "license": "by", - "license_version": "2.0", - "license_url": "https://creativecommons.org/licenses/by/2.0/", - "foreign_landing_url": "https://www.flickr.com/photos/18090920@N07/5264886972", # noqa - "detail_url": "http://api.openverse.engineering/v1/images/7c829a03-fb24-4b57-9b03-65f43ed19395", # noqa - "related_url": "http://api.openverse.engineering/v1/recommendations/images/7c829a03-fb24-4b57-9b03-65f43ed19395", # noqa - "height": 167, - "width": 500, - "attribution": "\"exam test\" by Sean MacEntee is licensed under CC-BY 2.0. To view a copy of this license, visit https://creativecommons.org/licenses/by/2.0/" # noqa - } -} - -image_detail_404_example = { - "application/json": { - "detail": "Not found." - } -} - -oembed_list_200_example = { +image_oembed_200_example = { "application/json": { - "version": 1, + "version": '1.0', "type": "photo", - "width": 500, - "height": 167, - "title": "exam test", - "author_name": "Sean MacEntee", - "author_url": "https://www.flickr.com/photos/18090920@N07", - "license_url": "https://creativecommons.org/licenses/by/2.0/" + "width": 1276, + "height": 1536, + "title": "Bust of Patroclus (photograph; calotype; salt print)", + "author_name": "William Henry Fox Talbot", + "author_url": None, + "license_url": "https://creativecommons.org/licenses/by-nc-nd/4.0/", } } -oembed_list_404_example = { +image_oembed_404_example = { "application/json": { "detail": "An internal server error occurred." } } -images_report_create_201_example = { +image_complain_201_example = { "application/json": { - "id": 10, - "identifier": "7c829a03-fb24-4b57-9b03-65f43ed19395", + "identifier": identifier, "reason": "mature", "description": "This image contains sensitive content" } } - -image_stats_200_example = { - "application/json": [ - { - "source_name": "flickr", - "image_count": 465809213, - "display_name": "Flickr", - "source_url": "https://www.flickr.com", - } - ] -} From a0012435bd6b04b31426246b2fb277646b226fa3 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 16:34:27 +0530 Subject: [PATCH 28/37] Define request response mappings and use them to run tests on the endpoints --- .../catalog/api/examples/__init__.py | 14 +++++++ openverse-api/test/examples_test.py | 39 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 openverse-api/test/examples_test.py diff --git a/openverse-api/catalog/api/examples/__init__.py b/openverse-api/catalog/api/examples/__init__.py index 550027607..9d6f55bf5 100644 --- a/openverse-api/catalog/api/examples/__init__.py +++ b/openverse-api/catalog/api/examples/__init__.py @@ -37,3 +37,17 @@ image_oembed_200_example, image_oembed_404_example, ) + +audio_mappings = { + audio_search_curl: audio_search_200_example, + audio_stats_curl: audio_stats_200_example, + audio_detail_curl: audio_detail_200_example, + audio_complain_curl: audio_complain_201_example, +} +image_mappings = { + image_search_curl: image_search_200_example, + image_stats_curl: image_stats_200_example, + image_detail_curl: image_detail_200_example, + image_complain_curl: image_complain_201_example, + image_oembed_curl: image_oembed_200_example, +} diff --git a/openverse-api/test/examples_test.py b/openverse-api/test/examples_test.py new file mode 100644 index 000000000..2c64937f5 --- /dev/null +++ b/openverse-api/test/examples_test.py @@ -0,0 +1,39 @@ +import json +import os +import subprocess + +import pytest + +from test.constants import API_URL + +os.environ['AUDIO_REQ_TOKEN'] = '' +os.environ['AUDIO_REQ_ORIGIN'] = API_URL +os.environ['AUDIO_REQ_IDX'] = '440a0240-8b20-49e2-a4e6-6fee550fcc41' + +from catalog.api.examples import ( # noqa | Set env vars before import + audio_mappings, + image_mappings, +) + + +def execute_request(request): + proc = subprocess.run(request, check=True, capture_output=True, shell=True) + return json.loads(proc.stdout) + + +@pytest.mark.parametrize( + 'in_val, out_val', + list(audio_mappings.items()) +) +def test_audio_success_examples(in_val, out_val): + res = execute_request(in_val) + assert res == out_val['application/json'] + + +@pytest.mark.parametrize( + 'in_val, out_val', + list(image_mappings.items()) +) +def test_image_success_examples(in_val, out_val): + res = execute_request(in_val) + assert res == out_val['application/json'] From 545916abd9c9be8a18ca2722874b7c7db245b34b Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 1 Sep 2021 17:05:20 +0530 Subject: [PATCH 29/37] Type hint `SerializerMethodField`s for Swagger --- openverse-api/catalog/api/serializers/audio_serializers.py | 2 +- openverse-api/catalog/api/serializers/image_serializers.py | 5 ++--- openverse-api/catalog/api/serializers/media_serializers.py | 2 ++ .../catalog/api/serializers/provider_serializers.py | 4 +++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openverse-api/catalog/api/serializers/audio_serializers.py b/openverse-api/catalog/api/serializers/audio_serializers.py index ee96f49ab..193126e65 100644 --- a/openverse-api/catalog/api/serializers/audio_serializers.py +++ b/openverse-api/catalog/api/serializers/audio_serializers.py @@ -192,5 +192,5 @@ class AudioWaveformSerializer(serializers.Serializer): ) @staticmethod - def get_len(obj): + def get_len(obj) -> int: return len(obj.get('points', [])) diff --git a/openverse-api/catalog/api/serializers/image_serializers.py b/openverse-api/catalog/api/serializers/image_serializers.py index f18927c0a..f4c0e4a11 100644 --- a/openverse-api/catalog/api/serializers/image_serializers.py +++ b/openverse-api/catalog/api/serializers/image_serializers.py @@ -1,4 +1,3 @@ -from django.urls import reverse from rest_framework import serializers from catalog.api.controllers.search_controller import get_sources @@ -214,10 +213,10 @@ class Meta: 'license_url', ] - def get_width(self, obj): + def get_width(self, obj) -> int: return self.context.get('width', obj.width) - def get_height(self, obj): + def get_height(self, obj) -> int: return self.context.get('height', obj.height) diff --git a/openverse-api/catalog/api/serializers/media_serializers.py b/openverse-api/catalog/api/serializers/media_serializers.py index 003b94e29..cb5f53a79 100644 --- a/openverse-api/catalog/api/serializers/media_serializers.py +++ b/openverse-api/catalog/api/serializers/media_serializers.py @@ -1,6 +1,7 @@ from collections import namedtuple from urllib.parse import urlparse +from drf_yasg.utils import swagger_serializer_method from rest_framework import serializers import catalog.api.licenses as license_helpers @@ -321,6 +322,7 @@ class MediaSerializer(serializers.Serializer): def get_license(self, obj): return obj.license.lower() + @swagger_serializer_method(serializer_or_field=serializers.URLField) def get_license_url(self, obj): if hasattr(obj, 'meta_data'): return license_helpers.get_license_url( diff --git a/openverse-api/catalog/api/serializers/provider_serializers.py b/openverse-api/catalog/api/serializers/provider_serializers.py index a996dbfba..39985b622 100644 --- a/openverse-api/catalog/api/serializers/provider_serializers.py +++ b/openverse-api/catalog/api/serializers/provider_serializers.py @@ -1,3 +1,4 @@ +from drf_yasg.utils import swagger_serializer_method from rest_framework import serializers from catalog.api.models import ContentProvider, SourceLogo @@ -33,6 +34,7 @@ class Meta: 'media_count', ] + @swagger_serializer_method(serializer_or_field=serializers.URLField) def get_logo_url(self, obj): try: source_logo = obj.sourcelogo @@ -43,6 +45,6 @@ def get_logo_url(self, obj): if request is not None: return request.build_absolute_uri(logo_path) - def get_media_count(self, obj): + def get_media_count(self, obj) -> int: source_counts = self.context.get('source_counts') return source_counts.get(obj.provider_identifier) From b263238fef8cdae638b7132b1ce43ee20d2d76ab Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 13:33:33 +0530 Subject: [PATCH 30/37] Move MD-in-Python out to its separate file --- openverse-api/catalog/api/docs/README.md | 116 +++++++++++++ openverse-api/catalog/urls/__init__.py | 200 ++--------------------- openverse-api/catalog/urls/swagger.py | 61 +++++++ 3 files changed, 188 insertions(+), 189 deletions(-) create mode 100644 openverse-api/catalog/api/docs/README.md create mode 100644 openverse-api/catalog/urls/swagger.py diff --git a/openverse-api/catalog/api/docs/README.md b/openverse-api/catalog/api/docs/README.md new file mode 100644 index 000000000..3c8bcc714 --- /dev/null +++ b/openverse-api/catalog/api/docs/README.md @@ -0,0 +1,116 @@ +# Introduction + +The Openverse API ('openverse-api') is a system that allows programmatic access +to public domain digital media. It is our ambition to index and catalog billions +of openly-licensed works, including articles, songs, videos, photographs, +paintings, and more. Using this API, developers will be able to access the +digital commons in their own applications. + +Please note that there is a rate limit of 5000 requests per day and 60 requests +per minute rate limit in place for anonymous users. This is fine for introducing +yourself to the API, but we strongly recommend that you obtain an API key as +soon as possible. Authorized clients have a higher rate limit of 10000 requests +per day and 100 requests per minute. Additionally, Openverse can give your key +an even higher limit that fits your application's needs. See the +[Register and Authenticate section](#section/Register-and-Authenticate) for +instructions on obtaining an API key. + +# Register and Authenticate + +## Register for a key +Before using the Openverse API, you need to register access via OAuth2. This can +be done using the `/v1/auth_tokens/register` endpoint. + +Example on how to register for a key: +```bash +$ curl \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"name": "My amazing project", "description": "To access Openverse API", "email": "zack.krida@automattic.com"}' \ + "https://api.openverse.engineering/v1/auth_tokens/register" +``` +If your request is successful, you will get a `client_id` and `client_secret`. + +Example of successful request: +```json +{ + "client_secret" : "YhVjvIBc7TuRJSvO2wIi344ez5SEreXLksV7GjalLiKDpxfbiM8qfUb5sNvcwFOhBUVzGNdzmmHvfyt6yU3aGrN6TAbMW8EOkRMOwhyXkN1iDetmzMMcxLVELf00BR2e", + "client_id" : "pm8GMaIXIhkjQ4iDfXLOvVUUcIKGYRnMlZYApbda", + "name" : "My amazing project" +} +``` + +## Authenticate +In order to use the Openverse API endpoints, you need to include access token in +the header. This can be done by exchanging your client credentials for a token +using the `/v1/auth_tokens/token/` endpoint. + +Example on how to authenticate using OAuth2: +```bash +$ curl \ + -X POST \ + -d "client_id=pm8GMaIXIhkjQ4iDfXLOvVUUcIKGYRnMlZYApbda&client_secret=YhVjvIBc7TuRJSvO2wIi344ez5SEreXLksV7GjalLiKDpxfbiM8qfUb5sNvcwFOhBUVzGNdzmmHvfyt6yU3aGrN6TAbMW8EOkRMOwhyXkN1iDetmzMMcxLVELf00BR2e&grant_type=client_credentials" \ + "https://api.openverse.engineering/v1/auth_tokens/token/" +``` +If your request is successful, you will get an access token. + +Example of successful request: +```json + { + "access_token" : "DLBYIcfnKfolaXKcmMC8RIDCavc2hW", + "scope" : "read write groups", + "expires_in" : 36000, + "token_type" : "Bearer" + } +``` + +Check your email for a verification link. After you have followed the link, your +API key will be activated. + +## Using Access Token +Include the `access_token` in the authorization header to use your key in your +future API requests. + +Example on how to make an authenticated request: +```bash +$ curl \ + -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" \ + "https://api.openverse.engineering/v1/images?q=test" +``` + +> **NOTE:** Your token will be throttled like an anonymous user until the email +> address has been verified. + +# Glossary + +| Term | Definition | +|-------------------|---| +| API | an abbreviation for Application Programming Interface | +| OAuth2 | an authorization framework that enables a third party application to get access to an HTTP service | +| access token | a private string that authorizes an application to make API requests | +| client ID | a publicly exposed string used by Openverse API to identify the application | +| client secret | a private string that authenticates the identity of the application to the Openverse API | +| CC | an abbreviation for Creative Commons | +| copyright | a type of intellectual property that gives the owner an exclusive right to reproduce, publish, sell or distribute content | +| mature content | any content that requires the audience to be 18 and older | +| sensitive content | any content that depicts graphic violence, adult content, and hostility or malice against others based on their race, religion, disability, sexual orientation, ethnicity and national origin | + +# Contribute + +We love pull requests! If you’re interested in +[contributing on Github](https://github.com/wordpress/openverse-api), here’s a +todo list to get started. + +- Read up about [Django REST Framework](https://www.django-rest-framework.org/), + which is the framework used to build Openverse API +- Read up about [drf-yasg](https://drf-yasg.readthedocs.io/en/stable/), which is + a tool used to generate real Swagger/OpenAPI 2.0 specifications +- Read up about Documentation Guidelines, which provides guidelines on how to + contribute to documentation, documentation styles and cheat sheet for drf-yasg +- Run the server locally by following this + [link](https://github.com/wordpress/openverse-api#running-the-server-locally) +- Update documentation or codebase +- Make sure the updates passed the automated tests in this + [file](https://github.com/wordpress/openverse-api/blob/master/.github/workflows/integration-tests.yml) +- Commit and push +- Create pull request diff --git a/openverse-api/catalog/urls/__init__.py b/openverse-api/catalog/urls/__init__.py index 2c036e586..7a141ef1e 100644 --- a/openverse-api/catalog/urls/__init__.py +++ b/openverse-api/catalog/urls/__init__.py @@ -13,14 +13,11 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -import rest_framework.permissions -from django.conf import settings + from django.conf.urls import include from django.contrib import admin from django.urls import path, re_path from django.views.generic import RedirectView -from drf_yasg import openapi -from drf_yasg.views import get_schema_view from rest_framework.routers import SimpleRouter from catalog.api.views.audio_views import AudioViewSet @@ -29,179 +26,18 @@ from catalog.api.utils.status_code_view import get_status_code_view from catalog.urls.auth_tokens import urlpatterns as auth_tokens_patterns - -description = """ -# Introduction -The Openverse API ('openverse-api') is a system -that allows programmatic access to public domain digital media. It is our -ambition to index and catalog billions of openly-licensed works, including -articles, songs, videos, photographs, paintings, and more. Using this API, -developers will be able to access the digital commons in their own -applications. - -Please note that there is a rate limit of 5000 requests per day and -60 requests per minute rate limit in place for anonymous users. This is fine -for introducing yourself to the API, but we strongly recommend that you obtain -an API key as soon as possible. Authorized clients have a higher rate limit -of 10000 requests per day and 100 requests per minute. Additionally, Openverse -can give your key an even higher limit that fits your application's -needs. See the Register and Authenticate section for instructions on obtaining -an API key. - -# Register and Authenticate - -## Register for a key -Before using the Openverse API, you need to register access via OAuth2. -This can be done using the `/v1/auth_tokens/register` endpoint. - -
-Example on how to register for a key - -``` -$ curl -X POST -H "Content-Type: application/json" -d '{"name": "My amazing project", "description": "To access Openverse API", "email": "zack.krida@automattic.com"}' https://api.openverse.engineering/v1/auth_tokens/register -``` - -
-If your request is succesful, you will get a `client_id` and `client_secret`. - -Example of successful request - -``` -{ - "client_secret" : "YhVjvIBc7TuRJSvO2wIi344ez5SEreXLksV7GjalLiKDpxfbiM8qfUb5sNvcwFOhBUVzGNdzmmHvfyt6yU3aGrN6TAbMW8EOkRMOwhyXkN1iDetmzMMcxLVELf00BR2e", - "client_id" : "pm8GMaIXIhkjQ4iDfXLOvVUUcIKGYRnMlZYApbda", - "name" : "My amazing project" -} -``` - -## Authenticate -In order to use the Openverse API endpoints, you need to include access token \ -in the header. -This can be done by exchanging your client credentials for a token using the \ -`v1/auth_tokens/token/` endpoint. - -
-Example on how to authenticate using OAuth2 - -``` -$ curl -X POST -d "client_id=pm8GMaIXIhkjQ4iDfXLOvVUUcIKGYRnMlZYApbda&client_secret=YhVjvIBc7TuRJSvO2wIi344ez5SEreXLksV7GjalLiKDpxfbiM8qfUb5sNvcwFOhBUVzGNdzmmHvfyt6yU3aGrN6TAbMW8EOkRMOwhyXkN1iDetmzMMcxLVELf00BR2e&grant_type=client_credentials" https://api.openverse.engineering/v1/auth_tokens/token/ -``` - -
-If your request is successful, you will get an access token. - -Example of successful request - -``` - { - "access_token" : "DLBYIcfnKfolaXKcmMC8RIDCavc2hW", - "scope" : "read write groups", - "expires_in" : 36000, - "token_type" : "Bearer" - } -``` - -Check your email for a verification link. After you have followed the link, \ -your API key will be activated. - -## Using Access Token -Include the `access_token` in the authorization header to use your key in \ -your future API requests. - -
-Example - -``` -$ curl -H "Authorization: Bearer DLBYIcfnKfolaXKcmMC8RIDCavc2hW" https://api.openverse.engineering/v1/images?q=test -``` -
-
- NOTE : Your token will be throttled like an anonymous user \ - until the email address has been verified. -
- -# Glossary - -#### Access Token -A private string that authorizes an application to make API requests - -#### API -An abbreviation for Application Programming Interface. - -#### CC -An abbreviation for Creative Commons. - -#### Client ID -A publicly exposed string used by Openverse API to identify the application. - -#### Client Secret -A private string that authenticates the identity of the application to the Openverse API. - -#### Copyright -A type of intellectual property that gives the owner an exclusive right to reproduce, publish, sell or distribute content. - -#### Mature content -Any content that requires the audience to be 18 and older. - -#### OAuth2 -An authorization framework that enables a third party application to get access to an HTTP service. - -#### Sensitive content -Any content that depicts graphic violence, adult content, and hostility or malice against others based on their race, religion, disability, sexual orientation, ethnicity and national origin. - -# Contribute - -We love pull requests! If you’re interested in [contributing on Github](https://github.com/wordpress/openverse-api), here’s a todo list to get started. - -- Read up about [Django REST Framework](https://www.django-rest-framework.org/), which is the framework used to build Openverse API -- Read up about [drf-yasg](https://drf-yasg.readthedocs.io/en/stable/), which is a tool used to generate real Swagger/OpenAPI 2.0 specifications -- Read up about Documentation Guidelines, which provides guidelines on how to contribute to documentation, documentation styles and cheat sheet for drf-yasg -- Run the server locally by following this [link](https://github.com/wordpress/openverse-api#running-the-server-locally) -- Update documentation or codebase -- Make sure the updates passed the automated tests in this [file](https://github.com/wordpress/openverse-api/blob/master/.github/workflows/integration-tests.yml) -- Commit and push -- Create pull request -""" # noqa - -# @todo: Reimplement logo once Openverse logomark is finalized -tos_url = "https://api.openverse.engineering/terms_of_service.html" -license_url = "https://github.com/" \ - "WordPress/openverse-api/blob/master/LICENSE" -logo_url = "https://raw.githubusercontent.com/" \ - "WordPress/openverse/master/brand/logo.svg" -schema_view = get_schema_view( - openapi.Info( - title="Openverse API", - default_version=settings.API_VERSION, - description=description, - contact=openapi.Contact(email="zack.krida@automattic.com"), - license=openapi.License(name="MIT License", url=license_url), - terms_of_service=tos_url, - x_logo={ - "url": logo_url, - "backgroundColor": "#fafafa" - } - ), - public=True, - permission_classes=(rest_framework.permissions.AllowAny,), -) - -cache_timeout = 0 if settings.DEBUG else 15 +from catalog.urls.swagger import urlpatterns as swagger_patterns discontinuation_message = { 'error': 'Gone', 'reason': 'This API endpoint has been discontinued.' } -router = SimpleRouter() -router.register('audio', AudioViewSet, basename='audio') -router.register('images', ImageViewSet, basename='image') - versioned_paths = [ path('rate_limit', CheckRates.as_view(), name='key_info'), path('auth_tokens/', include(auth_tokens_patterns)), - # Deprecated + # Deprecated, redirects to new URL path( 'sources', RedirectView.as_view(pattern_name='image-stats', permanent=True), @@ -227,13 +63,18 @@ name='thumbs' ), - # Discontinued + # Discontinued, return 410 Gone re_path( r'^link/', get_status_code_view(discontinuation_message, 410).as_view(), name='make-link' ), -] + router.urls +] + +router = SimpleRouter() +router.register('audio', AudioViewSet, basename='audio') +router.register('images', ImageViewSet, basename='image') +versioned_paths += router.urls urlpatterns = [ path('', RedirectView.as_view(pattern_name='root')), @@ -241,26 +82,7 @@ path('healthcheck', HealthCheck.as_view()), # Swagger documentation - re_path( - r'^swagger(?P\.json|\.yaml)$', - schema_view.without_ui(cache_timeout=None), - name='schema-json' - ), - re_path( - r'^swagger/$', - schema_view.with_ui('swagger', cache_timeout=cache_timeout), - name='schema-swagger-ui' - ), - re_path( - r'^redoc/$', - schema_view.with_ui('redoc', cache_timeout=cache_timeout), - name='schema-redoc' - ), - re_path( - r'^v1/$', - schema_view.with_ui('redoc', cache_timeout=cache_timeout), - name='root' - ), + path('', include(swagger_patterns)), # API path('v1/', include(versioned_paths)), diff --git a/openverse-api/catalog/urls/swagger.py b/openverse-api/catalog/urls/swagger.py new file mode 100644 index 000000000..b70ffdcef --- /dev/null +++ b/openverse-api/catalog/urls/swagger.py @@ -0,0 +1,61 @@ +from django.conf import settings +from django.urls import re_path +from drf_yasg import openapi +from drf_yasg.views import get_schema_view +from rest_framework.permissions import AllowAny + +description_path = settings.BASE_DIR.joinpath( + 'catalog', + 'api', + 'docs', + 'README.md', +) +with open(description_path, 'r') as description_file: + description = description_file.read() + +tos_url = "https://api.openverse.engineering/terms_of_service.html" +license_url = "https://github.com/" \ + "WordPress/openverse-api/blob/master/LICENSE" +logo_url = "https://raw.githubusercontent.com/" \ + "WordPress/openverse/master/brand/logo.svg" +schema_view = get_schema_view( + openapi.Info( + title="Openverse API", + default_version=settings.API_VERSION, + description=description, + contact=openapi.Contact(email="zack.krida@automattic.com"), + license=openapi.License(name="MIT License", url=license_url), + terms_of_service=tos_url, + x_logo={ + "url": logo_url, + "backgroundColor": "#fafafa" + } + ), + public=True, + permission_classes=(AllowAny,) +) + +cache_timeout = 0 if settings.DEBUG else 15 + +urlpatterns = [ + re_path( + r'^swagger(?P\.json|\.yaml)$', + schema_view.without_ui(cache_timeout=None), + name='schema-json' + ), + re_path( + r'^swagger/$', + schema_view.with_ui('swagger', cache_timeout=cache_timeout), + name='schema-swagger-ui' + ), + re_path( + r'^redoc/$', + schema_view.with_ui('redoc', cache_timeout=cache_timeout), + name='schema-redoc' + ), + re_path( + r'^v1/$', + schema_view.with_ui('redoc', cache_timeout=cache_timeout), + name='root' + ), +] From 5d9d667e9922aab710389615ffc32bfe65adb7ce Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 13:34:37 +0530 Subject: [PATCH 31/37] Add testing for the report endpoint --- openverse-api/test/audio_integration_test.py | 5 +++++ openverse-api/test/image_integration_test.py | 5 +++++ openverse-api/test/media_integration.py | 15 +++++++++++++++ openverse-api/test/v1_integration_test.py | 14 -------------- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/openverse-api/test/audio_integration_test.py b/openverse-api/test/audio_integration_test.py index 3b8f17324..61bac157c 100644 --- a/openverse-api/test/audio_integration_test.py +++ b/openverse-api/test/audio_integration_test.py @@ -17,6 +17,7 @@ detail, stats, thumb, + report, ) @@ -56,3 +57,7 @@ def test_audio_stats(): @pytest.mark.skip(reason='No images have audio set image yet') def test_audio_thumb(audio_fixture): thumb(audio_fixture) + + +def test_audio_report(audio_fixture): + report('audio', audio_fixture) diff --git a/openverse-api/test/image_integration_test.py b/openverse-api/test/image_integration_test.py index 9addb9df6..1a618b1e6 100644 --- a/openverse-api/test/image_integration_test.py +++ b/openverse-api/test/image_integration_test.py @@ -18,6 +18,7 @@ detail, stats, thumb, + report, ) @@ -58,6 +59,10 @@ def test_image_thumb(image_fixture): thumb(image_fixture) +def test_audio_report(image_fixture): + report('images', image_fixture) + + def test_oembed_endpoint_for_json(): params = { 'url': 'https://any.domain/any/path/29cb352c-60c1-41d8-bfa1-7d6f7d955f63', diff --git a/openverse-api/test/media_integration.py b/openverse-api/test/media_integration.py index 672b7727e..ad5758170 100644 --- a/openverse-api/test/media_integration.py +++ b/openverse-api/test/media_integration.py @@ -79,3 +79,18 @@ def thumb(fixture): thumbnail_response = requests.get(thumbnail_url) assert thumbnail_response.status_code == 200 assert thumbnail_response.headers["Content-Type"].startswith("image/") + + +def report(media_type, fixture): + test_id = fixture['results'][0]['id'] + response = requests.post( + f'{API_URL}/v1/{media_type}/{test_id}/report/', + { + 'reason': 'mature', + 'description': 'This item contains sensitive content', + }, + verify=False + ) + assert response.status_code == 201 + data = json.loads(response.text) + assert data['identifier'] == test_id diff --git a/openverse-api/test/v1_integration_test.py b/openverse-api/test/v1_integration_test.py index 0e8933aff..3274a4d4b 100644 --- a/openverse-api/test/v1_integration_test.py +++ b/openverse-api/test/v1_integration_test.py @@ -447,17 +447,3 @@ def test_related_image_search_page_consistency( related = recommendation_factory(image['id']) assert related['result_count'] > 0 assert len(related['results']) == 10 - - -def test_report_endpoint(image_fixture): - identifier = image_fixture['results'][0]['id'] - payload = { - 'reason': 'mature', - } - response = requests.post( - f'{API_URL}/v1/images/{identifier}/report/', - json=payload, verify=False) - assert response.status_code == 201 - data = json.loads(response.text) - assert data['identifier'] == identifier - return data From b745c3020d841bcca55fc161adc999d601e13847 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 13:36:18 +0530 Subject: [PATCH 32/37] Unskip thumbnail checks for audio ref. #171 --- openverse-api/test/audio_integration_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openverse-api/test/audio_integration_test.py b/openverse-api/test/audio_integration_test.py index 61bac157c..e9a06d785 100644 --- a/openverse-api/test/audio_integration_test.py +++ b/openverse-api/test/audio_integration_test.py @@ -54,7 +54,6 @@ def test_audio_stats(): stats('audio') -@pytest.mark.skip(reason='No images have audio set image yet') def test_audio_thumb(audio_fixture): thumb(audio_fixture) From a405b7997fbb576a681b69ef027ec37361fddf59 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 13:37:06 +0530 Subject: [PATCH 33/37] Remove unnecessary scheme replacement ref. 9d120d9 --- openverse-api/test/media_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openverse-api/test/media_integration.py b/openverse-api/test/media_integration.py index ad5758170..d4d0e076f 100644 --- a/openverse-api/test/media_integration.py +++ b/openverse-api/test/media_integration.py @@ -75,7 +75,7 @@ def stats(media_type, count_key='media_count'): def thumb(fixture): - thumbnail_url = fixture['results'][0]['thumbnail'].replace('https:', 'http:') + thumbnail_url = fixture['results'][0]['thumbnail'] thumbnail_response = requests.get(thumbnail_url) assert thumbnail_response.status_code == 200 assert thumbnail_response.headers["Content-Type"].startswith("image/") From 14914f717341b676512f71fb807ce22dfbd83ba5 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 13:37:22 +0530 Subject: [PATCH 34/37] Reformat code --- openverse-api/test/image_integration_test.py | 3 ++- openverse-api/test/v1_integration_test.py | 22 +++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/openverse-api/test/image_integration_test.py b/openverse-api/test/image_integration_test.py index 1a618b1e6..d0de8075b 100644 --- a/openverse-api/test/image_integration_test.py +++ b/openverse-api/test/image_integration_test.py @@ -4,10 +4,11 @@ """ import json +import xml.etree.ElementTree as ET from urllib.parse import urlencode + import pytest import requests -import xml.etree.ElementTree as ET from test.constants import API_URL from test.media_integration import ( diff --git a/openverse-api/test/v1_integration_test.py b/openverse-api/test/v1_integration_test.py index 3274a4d4b..e828c3478 100644 --- a/openverse-api/test/v1_integration_test.py +++ b/openverse-api/test/v1_integration_test.py @@ -3,19 +3,19 @@ designed. Run with the `pytest -s` command from this directory. """ -import requests import json -import pytest -import uuid import time -import catalog.settings +import uuid + +import pytest +import requests from django.db.models import Max from django.urls import reverse +import catalog.settings from catalog.api.licenses import LICENSE_GROUPS from catalog.api.models import Image, OAuth2Verification from catalog.api.utils.watermark import watermark - from test.constants import API_URL @@ -151,8 +151,8 @@ def test_auth_tokens_registration(): def test_auth_token_exchange(test_auth_tokens_registration): client_id = test_auth_tokens_registration['client_id'] client_secret = test_auth_tokens_registration['client_secret'] - token_exchange_request = f'client_id={client_id}&'\ - f'client_secret={client_secret}&'\ + token_exchange_request = f'client_id={client_id}&' \ + f'client_secret={client_secret}&' \ 'grant_type=client_credentials' headers = { 'content-type': "application/x-www-form-urlencoded", @@ -331,6 +331,7 @@ def _parameterized_search(**kwargs): assert response.status_code == 200 parsed = response.json() return parsed + return _parameterized_search @@ -339,8 +340,10 @@ def search_with_dead_links(search_factory): """ Here we pass filter_dead = False. """ + def _search_with_dead_links(**kwargs): return search_factory(filter_dead=False, **kwargs) + return _search_with_dead_links @@ -349,8 +352,10 @@ def search_without_dead_links(search_factory): """ Here we pass filter_dead = True. """ + def _search_without_dead_links(**kwargs): return search_factory(filter_dead=True, **kwargs) + return _search_without_dead_links @@ -440,7 +445,8 @@ def _parameterized_search(identifier, **kwargs): @pytest.mark.skip(reason="Generally, we don't paginate related images, so " "consistency is less of an issue.") def test_related_image_search_page_consistency( - recommendation, search_without_dead_links + recommendation, + search_without_dead_links ): initial_images = search_without_dead_links(q='*', page_size=10) for image in initial_images['results']: From f15b7363de815ef419185f85756c3406a9cd9c93 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 14:24:02 +0530 Subject: [PATCH 35/37] Make operation ID for OEmbed consistent --- openverse-api/catalog/api/docs/image_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openverse-api/catalog/api/docs/image_docs.py b/openverse-api/catalog/api/docs/image_docs.py index 58ade5fca..a35807a94 100644 --- a/openverse-api/catalog/api/docs/image_docs.py +++ b/openverse-api/catalog/api/docs/image_docs.py @@ -252,7 +252,7 @@ class ImageOembed: ] swagger_setup = { - 'operation_id': "oembed_list", + 'operation_id': "image_oembed", 'operation_description': desc, 'query_serializer': OembedRequestSerializer, 'responses': responses, From 683eb1744653701bbb92847edb5ec320ec262dc0 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Thu, 2 Sep 2021 15:35:24 +0530 Subject: [PATCH 36/37] Disable pagination on stats endpoint --- openverse-api/catalog/api/views/media_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openverse-api/catalog/api/views/media_views.py b/openverse-api/catalog/api/views/media_views.py index 1c00b9e4b..f03dad54a 100644 --- a/openverse-api/catalog/api/views/media_views.py +++ b/openverse-api/catalog/api/views/media_views.py @@ -83,7 +83,8 @@ def list(self, request, *_, **__): # Extra actions @action(detail=False, - serializer_class=ProviderSerializer) + serializer_class=ProviderSerializer, + pagination_class=None) def stats(self, *_, **__): source_counts = search_controller.get_sources(self.default_index) context = self.get_serializer_context() | { From b93edebbadd62bd06cade801c94cd1d9a98a5326 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Tue, 21 Sep 2021 22:55:46 +0530 Subject: [PATCH 37/37] Add examples in the help text for provider serializer --- .../catalog/api/serializers/provider_serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openverse-api/catalog/api/serializers/provider_serializers.py b/openverse-api/catalog/api/serializers/provider_serializers.py index 39985b622..db2a681ac 100644 --- a/openverse-api/catalog/api/serializers/provider_serializers.py +++ b/openverse-api/catalog/api/serializers/provider_serializers.py @@ -7,15 +7,15 @@ class ProviderSerializer(serializers.ModelSerializer): source_name = serializers.CharField( source='provider_identifier', - help_text='The source of the media.', + help_text='The source of the media, e.g. flickr', ) display_name = serializers.CharField( source='provider_name', - help_text='The name of content provider.', + help_text='The name of content provider, e.g. Flickr', ) source_url = serializers.URLField( source='domain_name', - help_text='The URL of the source.', + help_text='The URL of the source, e.g. https://www.flickr.com', ) logo_url = serializers.SerializerMethodField( help_text='The URL to a logo of the source.',