Shotgrid: reviving archived assets is broken

services/leecher/leecher/listener.py

@@ -28,12 +28,16 @@
 import shotgun_api3
 
 # TODO: remove hash in future since it is only used as backward compatibility
+# filters on only real events, starting with `Leeched`, not on `Recreated`
+# which have lower SG ID as they were tried, failed and should be retried
+# TODO filtering by 'leeched' or 'Leeched" doesnt work, by 'eeched' yes, wtf
 LAST_EVENT_QUERY = """query LastShotgridEvent($eventTopic: String!) {
-  events(last: 20, topics: [$eventTopic]) {
+  events(last: 3, topics: [$eventTopic], filter: "eeched") {  
     edges {
       node {
         hash
         summary
+        description
       }
     }
   }

services/processor/processor/processor.py

@@ -17,8 +17,9 @@
 import ayon_api
 import shotgun_api3
 
-from utils import get_logger
+from utils import get_logger, get_event_hash
 
+from constants import MissingParentError
 
 class ShotgridProcessor:
     _sg: shotgun_api3.Shotgun = None
@@ -226,6 +227,7 @@ def start_processing(self):
                     )
                     continue
 
+                failed = False
                 for handler in self.handlers_map.get(payload["action"], []):
                     # If theres any handler "subscribed" to this event type..
                     try:
@@ -244,8 +246,47 @@ def start_processing(self):
                             self,
                             payload,
                         )
-
+                    except MissingParentError:
+                        failed = True
+                        ayon_api.update_event(
+                            event["id"],
+                            status="failed",
+                            description=(
+                                "An error ocurred while processing"
+                                f"{event_id_text}, will be retried"
+                            ),
+                            payload={
+                                "message": traceback.format_exc(),
+                            },
+                            retries=999
+                        )
+                        if not payload.get("already_retried"):
+                            self.log.error(
+                                f"Reprocess handler {handler.__name__}, "
+                                "will be retried in new order",
+                            )
+                            payload["already_retried"] = True
+
+                            # to limit primary key violation
+                            new_event_hash = get_event_hash(
+                                "shotgrid.event",
+                                f"{payload['sg_payload']['id']}_dummy"
+                            )
+                            desc = (source_event['description'].
+                                    replace("Leeched", "Recreated"))
+                            ayon_api.dispatch_event(
+                                "shotgrid.event",
+                                sender=socket.gethostname(),
+                                payload=payload,
+                                summary=summary,
+                                description=desc,
+                                event_hash=new_event_hash
+                            )
+                        else:
+                            self.log.warning("Source event already failed, "
+                                             "won't be retried again.")
                     except Exception:
+                        failed = True
                         self.log.error(
                             f"Unable to process handler {handler.__name__}",
                             exc_info=True
@@ -262,14 +303,15 @@ def start_processing(self):
                             },
                         )
 
-                self.log.info(
-                    "Event has been processed... setting to finished!")
+                if not failed:
+                    self.log.info(
+                        "Event has been processed... setting to finished!")
 
-                ayon_api.update_event(
-                    event["id"],
-                    description=f"Event processed successfully{event_id_text}",
-                    status="finished",
-                )
+                    ayon_api.update_event(
+                        event["id"],
+                        description=f"Event processed successfully{event_id_text}",
+                        status="finished",
+                    )
 
             except Exception:
                 self.log.error(traceback.format_exc())

services/shotgrid_common/ayon_shotgrid_hub/__init__.py

@@ -306,6 +306,17 @@ def react_to_shotgrid_event(self, sg_event_meta):
                 f"Ignoring event, AYON project {self.project_name} not found.")
             return
 
+        # task revived only sends attribute_change event, before attribute
+        # change of parent assent AND finally entity_revival of Asset event
+        if (sg_event_meta["type"] == "attribute_change"
+            and sg_event_meta["attribute_name"] == "retirement_date"):
+            if sg_event_meta["entity_type"].lower() == "task":
+                self.log.info("changed to entity_revival")
+                sg_event_meta["type"] = "entity_revival"
+            else:
+                # do nothing for update retirement_date on non existing Asset
+                return
+
         match sg_event_meta["type"]:
             case "new_entity" | "entity_revival":
                 self.log.info(

services/shotgrid_common/ayon_shotgrid_hub/update_from_shotgrid.py

@@ -39,6 +39,7 @@
     SHOTGRID_TYPE_ATTRIB,  # Ayon Entity Attribute.
     SHOTGRID_REMOVED_VALUE,  # Value for removed entities.
     SG_RESTRICTED_ATTR_FIELDS,
+    MissingParentError
 )
 
 from utils import get_logger
@@ -172,8 +173,9 @@ def create_ay_entity_from_sg_event(
     if not ay_parent_entity:
         # This really should be an edge  ase, since any parent event would
         # happen before this... but hey
-        raise ValueError(
-            "Parent does not exist in Ayon, try doing a Project Sync.")
+        raise MissingParentError(
+            "Parent does not exist in Ayon, this event will be retried"
+            "after while. Hopefully parent will be already created.")
 
     if sg_ay_dict["type"].lower() == "task":
         ay_entity = ayon_entity_hub.add_new_task(
@@ -284,7 +286,7 @@ def update_ayon_entity_from_sg_event(
     )
 
     if not ay_entity:
-        raise ValueError("Unable to update a non existing entity.")
+        raise MissingParentError("Unable to update a non existing entity.")
 
     # make sure the entity is not immutable
     if (

services/shotgrid_common/constants.py

@@ -156,3 +156,22 @@
     "session_uuid",
     "created_at",
 ]
+
+
+class MissingParentError(Exception):
+    """This error could depend on order of processing.
+
+    Logic should capture this exception and dispatch source event again for
+    reprocessing. Source event couldn't be only failed as it would keep same
+    order. New event will be placed at the end of the queue.
+
+    This exception should be only raised if parent of the created/updated
+    object doesn't exist yet.
+
+    (Use case - task should be updated, but its parent Asset doesn't exist
+    yet.)
+
+    Source event payload should contain some additional key (like
+    `already_retried` that would protect from infinitive retries.
+    """
+    pass