Merge pull request #1 from jokiefer/feature/bulk-support

Feature/bulk support

CHANGELOG.rst

@@ -7,6 +7,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+[0.3.0] - 2023-07-10
+--------------------
+
+Added
+~~~~~
+
+* bulk operating for `add` and `delete` operations
+
+Fixed
+~~~~~
+
+* adds `check_resource_identifier_object` check on parser to check update operation correctly
+
+
 [0.2.0] - 2023-07-06
 --------------------
 

README.rst

@@ -19,7 +19,7 @@ See the `usage <https://drf-json-api-atomic-operations.readthedocs.io/en/latest/
 Implemented Features
 ~~~~~~~~~~~~~~~~~~~~
 
-* creating, updating, removing multiple resources in a single request (sequential db calls)
+* creating, updating, removing multiple resources in a single request (sequential db calls optional bulk db calls for create and delete)
 * `Updating To-One Relationships <https://jsonapi.org/ext/atomic/#auto-id-updating-to-one-relationships>`_
 * `Updating To-Many Relationships <https://jsonapi.org/ext/atomic/#auto-id-updating-to-many-relationships>`_
 * error reporting with json pointer to the concrete operation and the wrong attributes
@@ -29,5 +29,4 @@ ToDo
 ~~~~
 
 * permission handling
-* use django bulk operations to optimize db execution time
 * `local identity (lid) <https://jsonapi.org/ext/atomic/#operation-objects>`_ handling

atomic_operations/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 VERSION = __version__  # synonym

atomic_operations/parsers.py

@@ -103,6 +103,7 @@ def check_update_operation(self, idx, operation):
                 raise MissingPrimaryData(idx)
             elif not isinstance(data, dict):
                 raise InvalidPrimaryDataType(idx, "object")
+            self.check_resource_identifier_object(idx, data, operation["op"])
 
     def check_remove_operation(self, idx, ref):
         if not ref:

atomic_operations/views.py

@@ -24,6 +24,9 @@ class AtomicOperationView(APIView):
     #
     serializer_classes: Dict = {}
 
+    sequential = True
+    response_data: List[Dict] = []
+
     # TODO: proof how to check permissions for all operations
     # permission_classes = TODO
     # call def check_permissions for `add` operation
@@ -89,30 +92,92 @@ def get_serializer_context(self):
     def post(self, request, *args, **kwargs):
         return self.perform_operations(request.data)
 
+    def handle_sequential(self, serializer, operation_code):
+        if operation_code in ["add", "update", "update-relationship"]:
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
+            if operation_code != "update-relationship":
+                self.response_data.append(serializer.data)
+        else:
+            # remove
+            serializer.instance.delete()
+
+    def perform_bulk_create(self, bulk_operation_data):
+        objs = []
+        model_class = bulk_operation_data["serializer_collection"][0].Meta.model
+        for _serializer in bulk_operation_data["serializer_collection"]:
+            _serializer.is_valid(raise_exception=True)
+            instance = model_class(**_serializer.validated_data)
+            objs.append(instance)
+            self.response_data.append(
+                _serializer.__class__(instance=instance).data)
+        model_class.objects.bulk_create(
+            objs)
+
+    def perform_bulk_delete(self, bulk_operation_data):
+        obj_ids = []
+        for _serializer in bulk_operation_data["serializer_collection"]:
+            obj_ids.append(_serializer.instance.pk)
+            self.response_data.append(_serializer.data)
+        bulk_operation_data["serializer_collection"][0].Meta.model.objects.filter(
+            pk__in=obj_ids).delete()
+
+    def handle_bulk(self, serializer, current_operation_code, bulk_operation_data):
+        bulk_operation_data["serializer_collection"].append(serializer)
+        if bulk_operation_data["next_operation_code"] != current_operation_code or bulk_operation_data["next_resource_type"] != serializer.initial_data["type"]:
+            if current_operation_code == "add":
+                self.perform_bulk_create(bulk_operation_data)
+            elif current_operation_code == "delete":
+                self.perform_bulk_delete(bulk_operation_data)
+            else:
+                # TODO: update in bulk requires more logic cause it could be a partial update and every field differs pers instance.
+                # Then we can't do a bulk operation. This is only possible for instances which changes the same field(s).
+                # Maybe the anylsis of this takes longer than simple handling updates in sequential mode.
+                # For now we handle updates always in sequential mode
+                self.handle_sequential(
+                    bulk_operation_data["serializer_collection"][0], current_operation_code)
+            bulk_operation_data["serializer_collection"] = []
+
     def perform_operations(self, parsed_operations: List[Dict]):
-        response_data: List[Dict] = []
+        self.response_data = []  # reset local response data storage
+
+        bulk_operation_data = {
+            "serializer_collection": [],
+            "next_operation_code": "",
+            "next_resource_type": ""
+        }
+
         with atomic():
+
             for idx, operation in enumerate(parsed_operations):
-                op_code = next(iter(operation))
-                obj = operation[op_code]
-                # TODO: collect operations of same op_code and resource type to support bulk_create | bulk_update | filter(id__in=[1,2,3]).delete()
+                operation_code = next(iter(operation))
+                obj = operation[operation_code]
+
                 serializer = self.get_serializer(
                     idx=idx,
                     data=obj,
-                    operation_code="update" if op_code == "update-relationship" else op_code,
+                    operation_code="update" if operation_code == "update-relationship" else operation_code,
                     resource_type=obj["type"],
-                    partial=True if "update" in op_code else False
+                    partial=True if "update" in operation_code else False
                 )
-                if op_code in ["add", "update", "update-relationship"]:
-                    serializer.is_valid(raise_exception=True)
-                    serializer.save()
-                    # FIXME: check if it is just a relationship update
-                    if op_code == "update-relationship":
-                        # relation update. No response data
-                        continue
-                    response_data.append(serializer.data)
-                else:
-                    # remove
-                    serializer.instance.delete()
 
-        return Response(response_data, status=status.HTTP_200_OK if response_data else status.HTTP_204_NO_CONTENT)
+                if self.sequential:
+                    self.handle_sequential(serializer, operation_code)
+                else:
+                    is_last_iter = parsed_operations.__len__() == idx + 1
+                    if is_last_iter:
+                        bulk_operation_data["next_operation_code"] = ""
+                        bulk_operation_data["next_resource_type"] = ""
+                    else:
+                        next_operation = parsed_operations[idx + 1]
+                        bulk_operation_data["next_operation_code"] = next(
+                            iter(next_operation))
+                        bulk_operation_data["next_resource_type"] = next_operation[bulk_operation_data["next_operation_code"]]["type"]
+
+                    self.handle_bulk(
+                        serializer=serializer,
+                        current_operation_code=operation_code,
+                        bulk_operation_data=bulk_operation_data
+                    )
+
+        return Response(self.response_data, status=status.HTTP_200_OK if self.response_data else status.HTTP_204_NO_CONTENT)

docs/source/usage.rst

@@ -73,4 +73,19 @@ Now you can call the api like below.
             }
          }
       }]
-   }
+   }
+
+
+Bulk operating
+==============
+
+By default all operations are sequential db calls. This package provides also bulk operating for creating and deleting resources. To activate it you need to configure the following.
+
+
+.. code-block:: python
+   
+   from atomic_operations.views import AtomicOperationView
+
+   class ConcretAtomicOperationView(AtomicOperationView):
+
+      sequential = False

tests/test_parsers.py

@@ -271,6 +271,27 @@ def test_primary_data_without_id(self):
             }
         )
 
+        data = {
+            ATOMIC_OPERATIONS: [
+                {
+                    "op": "update",
+                    "data": {
+                        "type": "articles",
+                    }
+                }
+            ]
+        }
+        stream = BytesIO(json.dumps(data).encode("utf-8"))
+        self.assertRaisesRegex(
+            JsonApiParseError,
+            "The resource identifier object must contain an `id` member",
+            self.parser.parse,
+            **{
+                "stream": stream,
+                "parser_context": self.parser_context
+            }
+        )
+
     def test_primary_data(self):
         data = {
             ATOMIC_OPERATIONS: [

tests/test_views.py

@@ -198,6 +198,155 @@ def test_view_processing_with_valid_request(self):
         self.assertQuerysetEqual(RelatedModelTwo.objects.filter(pk__in=[1, 2]),
                                  BasicModel.objects.get(pk=2).to_many.all())
 
+    def test_bulk_view_processing_with_valid_request(self):
+        operations = [
+            {
+                "op": "add",
+                "data": {
+                    "type": "BasicModel",
+                    "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                    }
+                }
+            }, {
+                "op": "add",
+                "data": {
+                    "type": "BasicModel",
+                    "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                    }
+                }
+            }, {
+                "op": "add",
+                "data": {
+                    "type": "BasicModel",
+                    "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                    }
+                }
+            }, {
+                "op": "add",
+                "data": {
+                    "type": "BasicModel",
+                    "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                    }
+                }
+            }, {
+                "op": "add",
+                "data": {
+                    "type": "RelatedModel",
+                    "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                    }
+                }
+            }, {
+                "op": "update",
+                "data": {
+                    "id": "1",
+                    "type": "RelatedModel",
+                    "attributes": {
+                            "text": "JSON API paints my bikeshed!2"
+                    }
+                }
+            }
+        ]
+
+        data = {
+            ATOMIC_OPERATIONS: operations
+        }
+
+        response = self.client.post(
+            path="/bulk",
+            data=data,
+            content_type=ATOMIC_CONTENT_TYPE,
+
+            **{"HTTP_ACCEPT": ATOMIC_CONTENT_TYPE}
+        )
+
+        # check response
+        self.assertEqual(200, response.status_code)
+
+        expected_result = {
+            ATOMIC_RESULTS: [
+                {
+                    "data": {
+                        "id": "1",
+                        "type": "BasicModel",
+                        "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                        },
+                        "relationships": {
+                            "to_many": {'data': [], 'meta': {'count': 0}},
+                            "to_one": {'data': None},
+                        }
+                    }
+                },
+                {
+                    "data": {
+                        "id": "2",
+                        "type": "BasicModel",
+                        "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                        },
+                        "relationships": {
+                            "to_many": {'data': [], 'meta': {'count': 0}},
+                            "to_one": {'data': None},
+                        }
+                    }
+                }, {
+                    "data": {
+                        "id": "3",
+                        "type": "BasicModel",
+                        "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                        },
+                        "relationships": {
+                            "to_many": {'data': [], 'meta': {'count': 0}},
+                            "to_one": {'data': None},
+                        }
+                    }
+                },
+                {
+                    "data": {
+                        "id": "4",
+                        "type": "BasicModel",
+                        "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                        },
+                        "relationships": {
+                            "to_many": {'data': [], 'meta': {'count': 0}},
+                            "to_one": {'data': None},
+                        }
+                    }
+                },
+                {
+                    "data": {
+                        "id": "1",
+                        "type": "RelatedModel",
+                        "attributes": {
+                            "text": "JSON API paints my bikeshed!"
+                        }
+                    }
+                },
+                {
+                    "data": {
+                        "id": "1",
+                        "type": "RelatedModel",
+                        "attributes": {
+                            "text": "JSON API paints my bikeshed!2"
+                        }
+                    }
+                }
+            ]
+        }
+
+        self.assertDictEqual(expected_result,
+                             json.loads(response.content))
+
+        # check db content
+        self.assertEqual(4, BasicModel.objects.count())
+
     def test_parser_exception_with_pointer(self):
         operations = [
             {

tests/urls.py

@@ -1,7 +1,10 @@
 from django.urls import path
 
-from tests.views import ConcretAtomicOperationView
+from tests.views import BulkAtomicOperationView, ConcretAtomicOperationView
+
 
 urlpatterns = [
-    path("", ConcretAtomicOperationView.as_view())
+    path("", ConcretAtomicOperationView.as_view()),
+    path("bulk", BulkAtomicOperationView.as_view())
+
 ]

tests/views.py

@@ -12,5 +12,11 @@ class ConcretAtomicOperationView(AtomicOperationView):
         "update:BasicModel": BasicModelSerializer,
         "remove:BasicModel": BasicModelSerializer,
         "add:RelatedModel": RelatedModelSerializer,
+        "update:RelatedModel": RelatedModelSerializer,
         "add:RelatedModelTwo": RelatedModelTwoSerializer,
+
     }
+
+
+class BulkAtomicOperationView(ConcretAtomicOperationView):
+    sequential = False