Add support for webhook blocks with manual set-up

Resolves #8748

autogpt_platform/backend/backend/blocks/compass/triggers.py

@@ -1,14 +1,14 @@
-from typing import Literal
+from pydantic import BaseModel
+
 from backend.data.block import (
     Block,
     BlockCategory,
+    BlockManualWebhookConfig,
     BlockOutput,
     BlockSchema,
-    BlockWebhookConfig,
 )
-from backend.data.model import CredentialsField, CredentialsMetaInput, SchemaField
+from backend.data.model import SchemaField
 from backend.integrations.webhooks.simple_webhook_manager import CompassWebhookType
-from pydantic import BaseModel
 
 
 class Transcription(BaseModel):
@@ -27,22 +27,7 @@ class TranscriptionDataModel(BaseModel):
 
 class CompassAITriggerBlock(Block):
     class Input(BlockSchema):
-        class EventsFilter(BaseModel):
-            all: bool = True
-
         payload: TranscriptionDataModel = SchemaField(hidden=True)
-        events: EventsFilter = SchemaField(
-            description="Filter the events to be triggered on.",
-            default=EventsFilter(all=True),
-        )
-
-        credentials: CredentialsMetaInput[Literal["compass"], Literal["api_key"]] = (
-            CredentialsField(
-                provider="compass",
-                supported_credential_types={"api_key"},
-                description="The Compass AI integration can be used with any API key with sufficient permissions for the blocks it is used on.",
-            )
-        )
 
     class Output(BlockSchema):
         transcription: str = SchemaField(
@@ -56,12 +41,9 @@ def __init__(self):
             categories={BlockCategory.HARDWARE},
             input_schema=CompassAITriggerBlock.Input,
             output_schema=CompassAITriggerBlock.Output,
-            webhook_config=BlockWebhookConfig(
+            webhook_config=BlockManualWebhookConfig(
                 provider="compass",
                 webhook_type=CompassWebhookType.TRANSCRIPTION,
-                resource_format="",
-                event_filter_input="events",
-                event_format="transcription.{event}",
             ),
             test_input=[
                 {"input": "Hello, World!"},

autogpt_platform/backend/backend/data/block.py

@@ -188,7 +188,12 @@ class EmptySchema(BlockSchema):
 
 
 # --8<-- [start:BlockWebhookConfig]
-class BlockWebhookConfig(BaseModel):
+class BlockManualWebhookConfig(BaseModel):
+    """
+    Configuration model for webhook-triggered blocks on which
+    the user has to manually set up the webhook at the provider.
+    """
+
     provider: str
     """The service provider that the webhook connects to"""
 
@@ -199,26 +204,36 @@ class BlockWebhookConfig(BaseModel):
     Only for use in the corresponding `WebhooksManager`.
     """
 
-    resource_format: str
+    event_filter_input: str = ""
     """
-    Template string for the resource that a block instance subscribes to.
-    Fields will be filled from the block's inputs (except `payload`).
-
-    Example: `f"{repo}/pull_requests"` (note: not how it's actually implemented)
-
-    Only for use in the corresponding `WebhooksManager`.
+    Name of the block's event filter input.
+    Leave empty if the corresponding webhook doesn't have distinct event/payload types.
     """
 
-    event_filter_input: str
-    """Name of the block's event filter input."""
-
     event_format: str = "{event}"
     """
     Template string for the event(s) that a block instance subscribes to.
     Applied individually to each event selected in the event filter input.
 
     Example: `"pull_request.{event}"` -> `"pull_request.opened"`
     """
+
+
+class BlockWebhookConfig(BlockManualWebhookConfig):
+    """
+    Configuration model for webhook-triggered blocks for which
+    the webhook can be automatically set up through the provider's API.
+    """
+
+    resource_format: str
+    """
+    Template string for the resource that a block instance subscribes to.
+    Fields will be filled from the block's inputs (except `payload`).
+
+    Example: `f"{repo}/pull_requests"` (note: not how it's actually implemented)
+
+    Only for use in the corresponding `WebhooksManager`.
+    """
     # --8<-- [end:BlockWebhookConfig]
 
 
@@ -238,7 +253,7 @@ def __init__(
         disabled: bool = False,
         static_output: bool = False,
         block_type: BlockType = BlockType.STANDARD,
-        webhook_config: Optional[BlockWebhookConfig] = None,
+        webhook_config: Optional[BlockWebhookConfig | BlockManualWebhookConfig] = None,
     ):
         """
         Initialize the block with the given schema.
@@ -274,23 +289,33 @@ def __init__(
         self.execution_stats = {}
 
         if self.webhook_config:
-            # Enforce shape of webhook event filter
-            event_filter_field = self.input_schema.model_fields[
-                self.webhook_config.event_filter_input
-            ]
-            if not (
-                isinstance(event_filter_field.annotation, type)
-                and issubclass(event_filter_field.annotation, BaseModel)
-                and all(
-                    field.annotation is bool
-                    for field in event_filter_field.annotation.model_fields.values()
-                )
+            # Enforce presence of credentials field on auto-setup webhook blocks
+            if (
+                isinstance(self.webhook_config, BlockWebhookConfig)
+                and CREDENTIALS_FIELD_NAME not in self.input_schema.model_fields
             ):
-                raise NotImplementedError(
-                    f"{self.name} has an invalid webhook event selector: "
-                    "field must be a BaseModel and all its fields must be boolean"
+                raise TypeError(
+                    "credentials field is required on auto-setup webhook blocks"
                 )
 
+            # Enforce shape of webhook event filter, if present
+            if self.webhook_config.event_filter_input:
+                event_filter_field = self.input_schema.model_fields[
+                    self.webhook_config.event_filter_input
+                ]
+                if not (
+                    isinstance(event_filter_field.annotation, type)
+                    and issubclass(event_filter_field.annotation, BaseModel)
+                    and all(
+                        field.annotation is bool
+                        for field in event_filter_field.annotation.model_fields.values()
+                    )
+                ):
+                    raise NotImplementedError(
+                        f"{self.name} has an invalid webhook event selector: "
+                        "field must be a BaseModel and all its fields must be boolean"
+                    )
+
             # Enforce presence of 'payload' input
             if "payload" not in self.input_schema.model_fields:
                 raise TypeError(

autogpt_platform/backend/backend/data/graph.py

@@ -84,6 +84,8 @@ def is_triggered_by_event_type(self, event_type: str) -> bool:
             raise ValueError(f"Block #{self.block_id} not found for node #{self.id}")
         if not block.webhook_config:
             raise TypeError("This method can't be used on non-webhook blocks")
+        if not block.webhook_config.event_filter_input:
+            return True
         event_filter = self.input_default.get(block.webhook_config.event_filter_input)
         if not event_filter:
             raise ValueError(f"Event filter is not configured on node #{self.id}")

autogpt_platform/backend/backend/data/integrations.py

@@ -83,8 +83,10 @@ async def get_webhook(webhook_id: str) -> Webhook:
     return Webhook.from_db(webhook)
 
 
-async def get_all_webhooks(credentials_id: str) -> list[Webhook]:
+async def get_all_webhooks_by_creds(credentials_id: str) -> list[Webhook]:
     """⚠️ No `user_id` check: DO NOT USE without check in user-facing endpoints."""
+    if not credentials_id:
+        raise ValueError("credentials_id must not be empty")
     webhooks = await IntegrationWebhook.prisma().find_many(
         where={"credentialsId": credentials_id},
         include=INTEGRATION_WEBHOOK_INCLUDE,

autogpt_platform/backend/backend/integrations/webhooks/base.py

@@ -1,7 +1,7 @@
 import logging
 import secrets
 from abc import ABC, abstractmethod
-from typing import ClassVar, Generic, TypeVar
+from typing import ClassVar, Generic, Optional, TypeVar
 from uuid import uuid4
 
 from fastapi import Request
@@ -43,11 +43,11 @@ async def get_suitable_webhook(
         ):
             return webhook
         return await self._create_webhook(
-            user_id, credentials, webhook_type, resource, events
+            user_id, webhook_type, resource, events, credentials
         )
 
     async def prune_webhook_if_dangling(
-        self, webhook_id: str, credentials: Credentials
+        self, webhook_id: str, credentials: Optional[Credentials]
     ) -> bool:
         webhook = await integrations.get_webhook(webhook_id)
         if webhook.attached_nodes is None:
@@ -56,7 +56,8 @@ async def prune_webhook_if_dangling(
             # Don't prune webhook if in use
             return False
 
-        await self._deregister_webhook(webhook, credentials)
+        if credentials:
+            await self._deregister_webhook(webhook, credentials)
         await integrations.delete_webhook(webhook.id)
         return True
 
@@ -132,27 +133,39 @@ async def _deregister_webhook(
     async def _create_webhook(
         self,
         user_id: str,
-        credentials: Credentials,
         webhook_type: WT,
         resource: str,
         events: list[str],
+        credentials: Optional[Credentials] = None,
+        register: bool = True,
     ) -> integrations.Webhook:
+        if not app_config.platform_base_url:
+            raise MissingConfigError(
+                "PLATFORM_BASE_URL must be set to use Webhook functionality"
+            )
+
         id = str(uuid4())
         secret = secrets.token_hex(32)
         provider_name = self.PROVIDER_NAME
         ingress_url = (
             f"{app_config.platform_base_url}/api/integrations/{provider_name}"
             f"/webhooks/{id}/ingress"
         )
-        provider_webhook_id, config = await self._register_webhook(
-            credentials, webhook_type, resource, events, ingress_url, secret
-        )
+        if register:
+            if not credentials:
+                raise TypeError("credentials are required if register = True")
+            provider_webhook_id, config = await self._register_webhook(
+                credentials, webhook_type, resource, events, ingress_url, secret
+            )
+        else:
+            provider_webhook_id, config = "", {}
+
         return await integrations.create_webhook(
             integrations.Webhook(
                 id=id,
                 user_id=user_id,
                 provider=provider_name,
-                credentials_id=credentials.id,
+                credentials_id=credentials.id if credentials else "",
                 webhook_type=webhook_type,
                 resource=resource,
                 events=events,