Open API Updates AAH-57 (#611)

* Add OpenAPI view at /api/automation-hub/v3/openapi.json
* Also add openapi.yaml and redoc at /api/automation-hub/docs/
* Add swaggerui view too
* Add some drf_spect hooks to config
* Exclude /pulp and /_ui for now

* Add Meta.ref_name to ui.serializers.Serializer based... Serializers
ui.Serializer is metaclass that changes that populates the
ref name based on some heuristics. The generated ref_name
was conflicting with the drf_spectacular finds the serializer
name resulting in weirdness.

Issue: AAH-57

.travis/script.sh

@@ -35,29 +35,11 @@ fi
 
 cd ../pulp-openapi-generator
 
-./generate.sh pulpcore python
-pip install ./pulpcore-client
 ./generate.sh galaxy_ng python
 pip install ./galaxy_ng-client
-./generate.sh pulp_ansible python
-pip install ./pulp_ansible-client
 cd $TRAVIS_BUILD_DIR
 
 if [ "$TEST" = 'bindings' ]; then
-  python $TRAVIS_BUILD_DIR/.travis/test_bindings.py
-  cd ../pulp-openapi-generator
-  if [ ! -f $TRAVIS_BUILD_DIR/.travis/test_bindings.rb ]
-  then
-    exit
-  fi
-
-  rm -rf ./pulpcore-client
-
-  ./generate.sh pulpcore ruby 0
-  cd pulpcore-client
-  gem build pulpcore_client
-  gem install --both ./pulpcore_client-0.gem
-  cd ..
   rm -rf ./galaxy_ng-client
 
   ./generate.sh galaxy_ng ruby 0

CHANGES/57.feature

@@ -0,0 +1,45 @@
+Enable OpenAPI spec at cloud.redhat.com/api/automation-hub/v3/openapi.json
+
+Update docs and decorators on viewsets and serializers to generate correct
+spec.
+
+Modify pulpcore openapigenerator to include concrete hrefs in addition
+to {ansible_collection_href}/ style endpoints.
+
+Need to provide the existing pulp /pulp/api/v3/docs/ view and
+a new view at /api/automation-hub/v3/openapi.json
+- new viewset may need drf-spectacular tweaks
+
+Sub tasks:
+- Create a snapshot of the OpenAPI spec in CI.
+  - setup any useful tooling for validating/verifying the spec
+    - openapidiff ?
+- Enable swaggerui view (/v3/swagger/ ?)
+
+Potential problems:
+
+- May want/need to import pulpcore openapi generator utils, which may not be in plugin
+api
+
+Before:
+
+Pulp uses drf-spectacular
+
+A "live" generated version of the API is available at
+
+http://localhost:5001/pulp/api/v3/docs/api.json
+http://localhost:5001/pulp/api/v3/docs/api.yaml
+
+And a "redoc" view at:
+http://localhost:5001/pulp/api/v3/docs/
+
+Note some issues:
+
+- Lots of endpoints are in the form "{ansible_collection_import_href}"
+  - in theory, all endpoints should start with a "/" but even
+  when evaluated, the above is "ansible/ansible/v3/collections/artifacts"
+
+- schema objects are inconsistent named
+  - pulpcore has no prefix
+  - pulp_ansible has ansible. prefix
+  - galaxy_ng sometimes? has galaxy. prefix and sometimes Galaxy

galaxy_ng/app/api/v3/serializers/collection.py

@@ -128,6 +128,9 @@ class CollectionUploadSerializer(Serializer):
 
     sha256 = serializers.CharField(required=False, default=None)
 
+    class Meta():
+        ref_name = "CollectionUploadWithDownloadUrlSerializer"
+
     def to_internal_value(self, data):
         """Parse and validate collection filename."""
         data = super().to_internal_value(data)

galaxy_ng/app/api/v3/serializers/task.py

@@ -40,6 +40,7 @@ def get_repository(self, obj):
 
     class Meta:
         model = Task
+        ref_name = "Task"
         fields = (
             'pulp_id',
             'name',

galaxy_ng/app/common/openapi.py

@@ -0,0 +1,33 @@
+import logging
+
+log = logging.getLogger(__name__)
+log.debug('openapi autoschema')
+
+
+def preprocess_debug_logger(endpoints, **kwargs):
+    """Log and then return an iterable of (path, path_regex, method, callback) tuples"""
+
+    log.debug("kwargs: %s", repr(kwargs))
+
+    for path, path_regex, method, callback in endpoints:
+        log.debug('path=%s, path_regex=%s, method=%s, callback=%s',
+                  path, path_regex, method, callback)
+
+    return endpoints
+    # return [
+    #     (path, path_regex, method, callback) for path, path_regex, method, callback in endpoints
+    #     if log.debug('path=%s, path_regex=%s, method=%s, callback=%s',
+    #                  path, path_regex, method, callback)
+    # ]
+
+
+def preprocess_exclude_endpoints(endpoints, **kwargs):
+    """Return an iterable of (path, path_regex, method, callback) with some endpoints removed
+
+    For example, the default is to to remove '/pulp' and '/_ui/' api endpoints.
+    """
+
+    return [
+        (path, path_regex, method, callback) for path, path_regex, method, callback in endpoints
+        if not path.startswith('/pulp') and '/_ui/' not in path
+    ]

galaxy_ng/app/settings.py

@@ -82,3 +82,20 @@
 # ----------------------
 X_PULP_CONTENT_HOST = "localhost"
 X_PULP_CONTENT_PORT = 24816
+
+# This is example of how to set this via enviroment variables via dynaconf
+# SPECTACULAR_SETTINGS__TITLE = "Automation Hub API __TITLE"
+
+SPECTACULAR_SETTINGS = {
+    "TITLE": "Automation Hub API",
+    "DESCRIPTION": "Fetch, Upload, Organize, and Distribute Ansible Collections",
+    "VERSION": "v3",
+    "LICENSE": {
+        "name": "GPLv2+",
+        "url": "https://raw.githubusercontent.com/ansible/galaxy_ng/master/LICENSE",
+    },
+    'PREPROCESSING_HOOKS': ['galaxy_ng.app.common.openapi.preprocess_exclude_endpoints',
+                            'galaxy_ng.app.common.openapi.preprocess_debug_logger'],
+    "COMPONENT_SPLIT_REQUEST": True,
+    "dynaconf_merge": True,
+}

galaxy_ng/app/urls.py

@@ -5,6 +5,12 @@
 from galaxy_ng.app import customadmin as admin
 from galaxy_ng.ui import urls as ui_urls
 
+from drf_spectacular.views import (
+    SpectacularJSONAPIView,
+    SpectacularYAMLAPIView,
+    SpectacularRedocView,
+    SpectacularSwaggerView,
+)
 
 API_PATH_PREFIX = settings.GALAXY_API_PATH_PREFIX.strip("/")
 
@@ -18,3 +24,35 @@
     path("", include("django_prometheus.urls")),
     path(settings.ADMIN_SITE_URL, admin.site.urls),
 ]
+
+urlpatterns.append(
+    path(
+        f"{API_PATH_PREFIX}/v3/openapi.json",
+        SpectacularJSONAPIView.as_view(),
+        name="schema",
+    )
+)
+
+urlpatterns.append(
+    path(
+        f"{API_PATH_PREFIX}/v3/openapi.yaml",
+        SpectacularYAMLAPIView.as_view(),
+        name="schema-yaml",
+    )
+)
+
+urlpatterns.append(
+    path(
+        f"{API_PATH_PREFIX}/v3/redoc/",
+        SpectacularRedocView.as_view(),
+        name="schema-redoc",
+    )
+)
+
+urlpatterns.append(
+    path(
+        f"{API_PATH_PREFIX}/v3/swagger-ui/",
+        SpectacularSwaggerView.as_view(url_name='schema'),
+        name='swagger-ui',
+    )
+)