Merge pull request #513 from jbernal0019/master

Implement json source files for pipelines

chris_backend/pipelines/migrations/0011_pipelinesourcefile_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0 on 2023-05-17 03:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pipelines', '0010_pipelinesourcefile'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pipelinesourcefile',
+            name='type',
+            field=models.CharField(blank=True, choices=[('yaml', 'YAML file'), ('json', 'JSON file')], default='yaml', max_length=8),
+        ),
+    ]

chris_backend/pipelines/models.py

@@ -10,6 +10,9 @@
 from plugins.models import Plugin, PluginParameter
 
 
+PIPELINE_SOURCE_FILE_TYPE_CHOICES = [('yaml', 'YAML file'), ('json', 'JSON file')]
+
+
 class Pipeline(models.Model):
     creation_date = models.DateTimeField(auto_now_add=True)
     modification_date = models.DateTimeField(auto_now_add=True)
@@ -199,6 +202,8 @@ def source_file_path(instance, filename):
 class PipelineSourceFile(models.Model):
     creation_date = models.DateTimeField(auto_now_add=True)
     fname = models.FileField(max_length=512, upload_to=source_file_path, unique=True)
+    type = models.CharField(choices=PIPELINE_SOURCE_FILE_TYPE_CHOICES, default='yaml',
+                            max_length=8, blank=True)
     pipeline = models.OneToOneField(Pipeline, on_delete=models.CASCADE,
                                     related_name='source_file')
     owner = models.ForeignKey('auth.User', null=True, on_delete=models.SET_NULL)
@@ -225,7 +230,7 @@ class PipelineSourceFileFilter(FilterSet):
     class Meta:
         model = PipelineSourceFile
         fields = ['id', 'min_creation_date', 'max_creation_date', 'fname', 'fname_exact',
-                  'fname_icontains', 'owner_username']
+                  'fname_icontains', 'type', 'owner_username']
 
 
 class PluginPiping(models.Model):

chris_backend/pipelines/serializers.py

@@ -493,7 +493,7 @@ class PipelineSourceFileSerializer(serializers.HyperlinkedModelSerializer):
 
     class Meta:
         model = PipelineSourceFile
-        fields = ('url', 'id', 'creation_date', 'fname', 'fsize', 'file_resource',
+        fields = ('url', 'id', 'creation_date', 'fname', 'fsize', 'type', 'file_resource',
                   'pipeline', 'owner')
 
     def get_file_link(self, obj):
@@ -515,17 +515,27 @@ def create(self, validated_data):
 
     def validate(self, data):
         """
-        Overriden to validate the pipeline data in the source file.
+        Overriden to validate and transform the pipeline data in the source file to the
+        json canonical representation.
         """
-        pipeline_repr = self.read_pipeline_representation(data['fname'])
-        pipeline_repr = self.get_pipeline_canonical_representation(pipeline_repr)
+        type = data.get('type', 'yaml')
+        pipeline_repr = ''
+
+        if type == 'yaml':
+            pipeline_repr = self.read_yaml_pipeline_representation(data['fname'])
+            pipeline_repr = self.get_yaml_pipeline_canonical_representation(pipeline_repr)
+
+        elif type == 'json':
+            pipeline_repr = self.read_json_pipeline_representation(data['fname'])
+            pipeline_repr = self.get_json_pipeline_canonical_representation(pipeline_repr)
+
         data['pipeline'] = pipeline_repr
         return data
 
     @staticmethod
-    def read_pipeline_representation(pipeline_source_file):
+    def read_yaml_pipeline_representation(pipeline_source_file):
         """
-        Custom method to read the submitted pipeline source file.
+        Custom method to read the submitted yaml pipeline source file.
         """
         try:
             pipeline_repr = yaml.safe_load(pipeline_source_file.read().decode())
@@ -536,10 +546,23 @@ def read_pipeline_representation(pipeline_source_file):
         return pipeline_repr
 
     @staticmethod
-    def get_pipeline_canonical_representation(pipeline_repr):
+    def read_json_pipeline_representation(pipeline_source_file):
+        """
+        Custom method to read the submitted json pipeline source file.
+        """
+        try:
+            pipeline_repr = json.loads(pipeline_source_file.read().decode())
+            pipeline_source_file.seek(0)
+        except Exception:
+            error_msg = "Invalid json representation file."
+            raise serializers.ValidationError({'fname': [error_msg]})
+        return pipeline_repr
+
+    @staticmethod
+    def get_yaml_pipeline_canonical_representation(pipeline_repr):
         """
-        Custom method to convert the submitted pipeline representation to the canonical
-        JSON representation.
+        Custom method to convert the submitted yaml pipeline representation to the
+        canonical JSON representation.
         """
         plugin_tree = []
         for node in pipeline_repr.get('plugin_tree', []):
@@ -574,6 +597,16 @@ def get_pipeline_canonical_representation(pipeline_repr):
         pipeline_repr['plugin_tree'] = json.dumps(plugin_tree)
         return pipeline_repr
 
+    @staticmethod
+    def get_json_pipeline_canonical_representation(pipeline_repr):
+        """
+        Custom method to convert the submitted json pipeline representation to the
+        canonical JSON representation.
+        """
+        plugin_tree = pipeline_repr.get('plugin_tree', [])
+        pipeline_repr['plugin_tree'] = json.dumps(plugin_tree)
+        return pipeline_repr
+
 class DefaultPipingStrParameterSerializer(serializers.HyperlinkedModelSerializer):
     previous_plugin_piping_id = serializers.ReadOnlyField(
         source='plugin_piping.previous_id')

chris_backend/pipelines/tests/test_serializers.py

@@ -1,9 +1,11 @@
 
 import logging
 import io
-import yaml
+import json
 from unittest import mock
 
+import yaml
+
 from django.test import TestCase
 from django.contrib.auth.models import User
 from django.conf import settings
@@ -565,7 +567,7 @@ class PipelineSourceFileSerializerTests(SerializerTests):
 
     def setUp(self):
         super(PipelineSourceFileSerializerTests, self).setUp()
-        self.pipeline_str = """
+        self.yaml_pipeline_str = """
         name: TestPipeline
         locked: false
         plugin_tree:
@@ -581,20 +583,34 @@ def setUp(self):
           plugin_parameter_defaults:
             plugininstances: simpledsapp1,simpledsapp2
         """
-
-    def test_read_pipeline_representation(self):
-        """
-        Test whether custom read_pipeline_representation method returns an appropriate
+        self.json_pipeline_str = '{"name": "TestPipeline", "locked": false,' \
+                                 '"plugin_tree": [' \
+                                 '{"title": "simpledsapp1",' \
+                                 '"plugin_name": "simpledsapp", ' \
+                                 '"plugin_version": "0.1","previous": null},' \
+                                 '{"title": "simpledsapp2",' \
+                                 '"plugin_name": "simpledsapp",' \
+                                 '"plugin_version": "0.1","previous": "simpledsapp1"},' \
+                                 '{"title": "join",' \
+                                 '"plugin_name": "ts_copy",' \
+                                 '"plugin_version": "0.1","previous": "simpledsapp1",' \
+                                 '"plugin_parameter_defaults": [' \
+                                 '{"name": "plugininstances",' \
+                                 '"default": "simpledsapp1,simpledsapp2"}]}]}'
+
+    def test_read_yaml_pipeline_representation(self):
+        """
+        Test whether custom read_yaml_pipeline_representation method returns an appropriate
         yaml pipeline representation from an uploaded yaml representation file.
         """
-        pipeline_repr = yaml.safe_load(self.pipeline_str)
-        with io.BytesIO(self.pipeline_str.encode()) as f:
-            self.assertEqual(PipelineSourceFileSerializer.read_pipeline_representation(f),
+        pipeline_repr = yaml.safe_load(self.yaml_pipeline_str)
+        with io.BytesIO(self.yaml_pipeline_str.encode()) as f:
+            self.assertEqual(PipelineSourceFileSerializer.read_yaml_pipeline_representation(f),
                              pipeline_repr)
 
-    def test_read_pipeline_representation_raises_validation_error_if_invalid_yaml_file(self):
+    def test_read_yaml_pipeline_representation_raises_validation_error_if_invalid_yaml_file(self):
         """
-        Test whether custom read_pipeline_representation method raises ValidationError if
+        Test whether custom read_yaml_pipeline_representation method raises ValidationError if
         uploaded yaml representation file is invalid.
         """
         pipeline_str = """
@@ -610,26 +626,54 @@ def test_read_pipeline_representation_raises_validation_error_if_invalid_yaml_fi
         """
         with self.assertRaises(serializers.ValidationError):
             with io.BytesIO(pipeline_str.encode()) as f:
-                PipelineSourceFileSerializer.read_pipeline_representation(f)
+                PipelineSourceFileSerializer.read_yaml_pipeline_representation(f)
+
+    def test_read_json_pipeline_representation(self):
+        """
+        Test whether custom read_json_pipeline_representation method returns an appropriate
+        json pipeline representation from an uploaded json representation file.
+        """
+        pipeline_repr = json.loads(self.json_pipeline_str)
+        with io.BytesIO(self.json_pipeline_str.encode()) as f:
+            self.assertEqual(PipelineSourceFileSerializer.read_json_pipeline_representation(f),
+                             pipeline_repr)
 
-    def test_get_pipeline_canonical_representation(self):
+    def test_read_json_pipeline_representation_raises_validation_error_if_invalid_json_file(self):
         """
-        Test whether custom get_pipeline_canonical_representation method returns an
+        Test whether custom read_json_pipeline_representation method raises
+        ValidationError if uploaded json representation file is invalid.
+        """
+        pipeline_str = '{"name": "TestPipeline", "locked": false,' \
+                         '"plugin_tree": [' \
+                         '{"title": "simpledsapp1",' \
+                         '"plugin_name": "simpledsapp", ' \
+                         '"plugin_version": "0.1","previous": null},' \
+                         '{"title": "simpledsapp2",' \
+                         '"plugin_name": "simpledsapp",' \
+                         '"plugin_version": "0.1","previous": "simpledsapp1"}}'
+        with self.assertRaises(serializers.ValidationError):
+            with io.BytesIO(pipeline_str.encode()) as f:
+                PipelineSourceFileSerializer.read_json_pipeline_representation(f)
+
+    def test_get_yaml_pipeline_canonical_representation(self):
+        """
+        Test whether custom get_yaml_pipeline_canonical_representation method returns an
         appropriate canonical JSON pipeline representation from the yaml representation.
         """
-        pipeline_repr = yaml.safe_load(self.pipeline_str)
+        pipeline_repr = yaml.safe_load(self.yaml_pipeline_str)
         canonical_repr = {'name': 'TestPipeline',
                           'locked': False,
                           'plugin_tree': '[{"title": "simpledsapp1", "plugin_name": "simpledsapp", "plugin_version": "0.1", "previous": null}, '
                                          '{"title": "simpledsapp2", "plugin_name": "simpledsapp", "plugin_version": "0.1", "previous": "simpledsapp1"}, '
                                          '{"title": "join", "plugin_name": "ts_copy", "plugin_version": "0.1", "previous": "simpledsapp1", '
                                          '"plugin_parameter_defaults": [{"name": "plugininstances", "default": "simpledsapp1,simpledsapp2"}]}]'}
-        self.assertEqual(PipelineSourceFileSerializer.get_pipeline_canonical_representation(pipeline_repr), canonical_repr)
+        self.assertEqual(PipelineSourceFileSerializer.get_yaml_pipeline_canonical_representation(pipeline_repr),
+                         canonical_repr)
 
 
-    def test_get_pipeline_canonical_representation_raises_validation_error_if_missing_node_plugin(self):
+    def test_get_yaml_pipeline_canonical_representation_raises_validation_error_if_missing_node_plugin(self):
         """
-        Test whether custom get_pipeline_canonical_representation method raises
+        Test whether custom get_yaml_pipeline_canonical_representation method raises
         ValidationError if a node is missing its plugin.
         """
         pipeline_str = """
@@ -641,11 +685,11 @@ def test_get_pipeline_canonical_representation_raises_validation_error_if_missin
         """
         pipeline_repr = yaml.safe_load(pipeline_str)
         with self.assertRaises(serializers.ValidationError):
-            PipelineSourceFileSerializer.get_pipeline_canonical_representation(pipeline_repr)
+            PipelineSourceFileSerializer.get_yaml_pipeline_canonical_representation(pipeline_repr)
 
-    def test_get_pipeline_canonical_representation_raises_validation_error_if_missing_plugin_name_or_version(self):
+    def test_get_yaml_pipeline_canonical_representation_raises_validation_error_if_missing_plugin_name_or_version(self):
         """
-        Test whether custom get_pipeline_canonical_representation method raises
+        Test whether custom get_yaml_pipeline_canonical_representation method raises
         ValidationError if a node is missing its plugin name or version.
         """
         pipeline_str = """
@@ -658,4 +702,19 @@ def test_get_pipeline_canonical_representation_raises_validation_error_if_missin
         """
         pipeline_repr = yaml.safe_load(pipeline_str)
         with self.assertRaises(serializers.ValidationError):
-            PipelineSourceFileSerializer.get_pipeline_canonical_representation(pipeline_repr)
+            PipelineSourceFileSerializer.get_yaml_pipeline_canonical_representation(pipeline_repr)
+
+    def test_get_json_pipeline_canonical_representation(self):
+        """
+        Test whether custom get_json_pipeline_canonical_representation method returns an
+        appropriate canonical JSON pipeline representation from the json representation.
+        """
+        pipeline_repr = json.loads(self.json_pipeline_str)
+        canonical_repr = {'name': 'TestPipeline',
+                          'locked': False,
+                          'plugin_tree': '[{"title": "simpledsapp1", "plugin_name": "simpledsapp", "plugin_version": "0.1", "previous": null}, '
+                                         '{"title": "simpledsapp2", "plugin_name": "simpledsapp", "plugin_version": "0.1", "previous": "simpledsapp1"}, '
+                                         '{"title": "join", "plugin_name": "ts_copy", "plugin_version": "0.1", "previous": "simpledsapp1", '
+                                         '"plugin_parameter_defaults": [{"name": "plugininstances", "default": "simpledsapp1,simpledsapp2"}]}]'}
+        self.assertEqual(PipelineSourceFileSerializer.get_json_pipeline_canonical_representation(pipeline_repr),
+                         canonical_repr)

chris_backend/pipelines/views.py

@@ -179,7 +179,7 @@ def list(self, request, *args, **kwargs):
         query_list = [reverse('pipelinesourcefile-list-query-search', request=request)]
         response = services.append_collection_querylist(response, query_list)
         # append write template
-        template_data = {'fname': ''}
+        template_data = {'type': '', 'fname': ''}
         return services.append_collection_template(response, template_data)