Sync with template repo

.github/workflows/_tests.yml

@@ -0,0 +1,23 @@
+name: Run tests
+on:
+  workflow_call
+
+jobs:
+  run_unit_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10' 
+      - name: Install pip
+        run: python -m ensurepip --upgrade
+      - name: Install pip tools
+        run: pip install pip-tools
+      - name: Install compile dependencies
+        run: pip-compile --output-file=requirements.txt requirements-base.in requirements-dev.in requirements.in
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Run unit tests
+        run: pytest

.github/workflows/main.yaml

@@ -23,16 +23,20 @@ jobs:
           echo "tf_file_dev=environments/dev/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT
           echo "tf_file_stage=environments/stage/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT
           echo "tf_file_prod=environments/prod/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT
-
+
+  run_unit_tests:
+    uses: ./.github/workflows/_tests.yml
+
   build:
     uses: PADAS/gundi-workflows/.github/workflows/build_docker.yml@v2
-    needs: vars
+    needs: [run_unit_tests, vars]
     with:
       repository: ${{ needs.vars.outputs.repository }}
       tag: ${{ needs.vars.outputs.tag }}
       workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}}
       service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }}
 
+
   deploy_dev:
     uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2
     if: startsWith(github.ref, 'refs/heads/main')

.github/workflows/pr.yaml

@@ -0,0 +1,7 @@
+name: Test integration
+on:
+  pull_request
+
+jobs:
+  pr_unit_tests:
+    uses: ./.github/workflows/_tests.yml

README.md

@@ -403,3 +403,57 @@ Sample configuration in Gundi:
 """
 ```
 Notice: This can also be combined with Dynamic Schema and JSON Transformations. In that case the hex string will be parsed first, adn then the JQ filter can be applied to the extracted data.
+
+### Custom UI for configurations (ui schema)
+It's possible to customize how the forms for configurations are displayed in the Gundi portal. 
+To do that, use `FieldWithUIOptions` in your models. The `UIOptions` and `GlobalUISchemaOptions` will allow you to customize the appearance of the fields in the portal by setting any of the ["ui schema"](https://rjsf-team.github.io/react-jsonschema-form/docs/api-reference/uiSchema) supported options.
+
+```python
+# Example
+import pydantic
+from app.services.utils import FieldWithUIOptions, GlobalUISchemaOptions, UIOptions
+from .core import AuthActionConfiguration, PullActionConfiguration
+
+
+class AuthenticateConfig(AuthActionConfiguration):
+    email: str  # This will be rendered with default widget and settings
+    password: pydantic.SecretStr = FieldWithUIOptions(
+        ...,
+        format="password",
+        title="Password",
+        description="Password for the Global Forest Watch account.",
+        ui_options=UIOptions(
+            widget="password",  # This will be rendered as a password input hiding the input
+        )
+    )
+    ui_global_options = GlobalUISchemaOptions(
+        order=["email", "password"],  # This will set the order of the fields in the form
+    )
+
+
+class MyPullActionConfiguration(PullActionConfiguration):
+    lookback_days: int = FieldWithUIOptions(
+        10,
+        le=30,
+        ge=1,
+        title="Data lookback days",
+        description="Number of days to look back for data.",
+        ui_options=UIOptions(
+            widget="range",  # This will be rendered ad a range slider
+        )
+    )
+    force_fetch: bool = FieldWithUIOptions(
+        False,
+        title="Force fetch",
+        description="Force fetch even if in a quiet period.",
+        ui_options=UIOptions(
+            widget="radio", # This will be rendered as a radio button
+        )
+    )
+    ui_global_options = GlobalUISchemaOptions(
+        order=[
+            "lookback_days",
+            "force_fetch",
+        ],
+    )
+```

app/actions/core.py

@@ -1,9 +1,12 @@
 import importlib
 import inspect
+from typing import Optional
+
 from pydantic import BaseModel
+from app.services.utils import UISchemaModelMixin
 
 
-class ActionConfiguration(BaseModel):
+class ActionConfiguration(UISchemaModelMixin, BaseModel):
     pass
 
 

app/conftest.py

@@ -1,15 +1,13 @@
 import asyncio
 import datetime
 import json
-
 import pydantic
 import pytest
 from unittest.mock import MagicMock
 from app import settings
 from gcloud.aio import pubsub
-from gundi_core.schemas.v2 import Integration, IntegrationActionConfiguration, IntegrationActionSummary
+from gundi_core.schemas.v2 import Integration
 from gundi_core.events import (
-    SystemEventBaseModel,
     IntegrationActionCustomLog,
     CustomActivityLog,
     IntegrationActionStarted,
@@ -28,9 +26,9 @@
     CustomWebhookLog,
     LogLevel
 )
-
 from app.actions import PullActionConfiguration
-from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload
+from app.services.utils import GlobalUISchemaOptions, FieldWithUIOptions, UIOptions
+from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload, WebhookConfiguration
 
 
 class AsyncMock(MagicMock):
@@ -107,9 +105,9 @@ def integration_v2():
                          'value': 'auth'}, 'data': {'token': 'testtoken2a97022f21732461ee103a08fac8a35'}}],
          'additional': {},
          'default_route': {'id': '5abf3845-7c9f-478a-bc0f-b24d87038c4b', 'name': 'Gundi X Provider - Default Route'},
-         'status': {'id': 'mockid-b16a-4dbd-ad32-197c58aeef59', 'is_healthy': True,
-                    'details': 'Last observation has been delivered with success.',
-                    'observation_delivered_24hrs': 50231, 'last_observation_delivered_at': '2023-03-31T11:20:00+0200'}}
+         'status': 'healthy',
+         'status_details': '',
+        }
     )
 
 
@@ -139,6 +137,14 @@ def integration_v2_with_webhook():
                             "allowed_devices_list": {"title": "Allowed Devices List", "type": "array", "items": {}},
                             "deduplication_enabled": {"title": "Deduplication Enabled", "type": "boolean"}},
                         "required": ["allowed_devices_list", "deduplication_enabled"]
+                    },
+                    "ui_schema": {
+                        "allowed_devices_list": {
+                            "ui:widget": "select"
+                        },
+                        "deduplication_enabled": {
+                            "ui:widget": "radio"
+                        }
                     }
                 }
             },
@@ -163,13 +169,8 @@ def integration_v2_with_webhook():
             },
             "additional": {},
             "default_route": None,
-            "status": {
-                "id": "mockid-b16a-4dbd-ad32-197c58aeef59",
-                "is_healthy": True,
-                "details": "Last observation has been delivered with success.",
-                "observation_delivered_24hrs": 50231,
-                "last_observation_delivered_at": "2023-03-31T11:20:00+0200"
-            }
+            "status": "healthy",
+            "status_details": "",
         }
     )
 
@@ -218,6 +219,17 @@ def integration_v2_with_webhook_generic():
                                 "description": "Output type for the transformed data: 'obv' or 'event'"
                             }
                         }
+                    },
+                    "ui_schema": {
+                        "jq_filter": {
+                            "ui:widget": "textarea"
+                        },
+                        "json_schema": {
+                            "ui:widget": "textarea"
+                        },
+                        "output_type": {
+                            "ui:widget": "text"
+                        }
                     }
                 }
             },
@@ -455,13 +467,8 @@ def integration_v2_with_webhook_generic():
             },
             "additional": {},
             "default_route": None,
-            "status": {
-                "id": "mockid-b16a-4dbd-ad32-197c58aeef59",
-                "is_healthy": True,
-                "details": "Last observation has been delivered with success.",
-                "observation_delivered_24hrs": 50231,
-                "last_observation_delivered_at": "2023-03-31T11:20:00+0200"
-            }
+            "status": "healthy",
+            "status_details": "",
         }
     )
 
@@ -898,7 +905,30 @@ def mock_publish_event(gcp_pubsub_publish_response):
 
 
 class MockPullActionConfiguration(PullActionConfiguration):
-    lookback_days: int = 10
+    lookback_days: int = FieldWithUIOptions(
+        30,
+        le=30,
+        ge=1,
+        title="Data lookback days",
+        description="Number of days to look back for data.",
+        ui_options=UIOptions(
+            widget="range",
+        )
+    )
+    force_fetch: bool = FieldWithUIOptions(
+        False,
+        title="Force fetch",
+        description="Force fetch even if in a quiet period.",
+        ui_options=UIOptions(
+            widget="select",
+        )
+    )
+    ui_global_options = GlobalUISchemaOptions(
+        order=[
+            "lookback_days",
+            "force_fetch",
+        ],
+    )
 
 
 @pytest.fixture
@@ -1172,9 +1202,21 @@ class MockWebhookPayloadModel(WebhookPayload):
     lon: float
 
 
-class MockWebhookConfigModel(pydantic.BaseModel):
-    allowed_devices_list: list
-    deduplication_enabled: bool
+class MockWebhookConfigModel(WebhookConfiguration):
+    allowed_devices_list: list = FieldWithUIOptions(
+        ...,
+        title="Allowed Devices List",
+        ui_options=UIOptions(
+            widget="list",
+        )
+    )
+    deduplication_enabled: bool = FieldWithUIOptions(
+        ...,
+        title="Deduplication Enabled",
+        ui_options=UIOptions(
+            widget="radio",
+        )
+    )
 
 
 @pytest.fixture

app/services/gundi.py

@@ -52,11 +52,14 @@ async def send_events_to_gundi(events: List[dict], **kwargs) -> dict:
 
 
 @stamina.retry(on=httpx.HTTPError, wait_initial=1.0, wait_jitter=5.0, wait_max=32.0)
-async def send_event_attachments_to_gundi(event_id: str, attachments: List[bytes], **kwargs) -> dict:
+async def send_event_attachments_to_gundi(event_id: str, attachments: List[tuple], **kwargs) -> dict:
     """
     Send Event Attachments to Gundi using the REST API v2
     :param event_id: Created event in which the attachments are going to be linked
-    :param attachments: A list of attachments (in bytes)
+    :param attachments: A list of attachments (tuples with filename, file in bytes). Example:
+    filename = 'example.png'
+    file_in_bytes = open(filename, 'rb')
+    attachments = [(filename, file_in_bytes)]
     :param kwargs: integration_id: The UUID of the related integration
     :return: A dict with the response from the API
     """

app/services/self_registration.py

@@ -14,8 +14,6 @@
 
 
 async def register_integration_in_gundi(gundi_client, type_slug=None, service_url=None):
-    #from ..webhooks.configurations import LiquidTechPayload
-    #print(GenericJsonTransformConfig.schema_json())
     # Prepare the integration name and value
     integration_type_slug = type_slug or INTEGRATION_TYPE_SLUG
     if not integration_type_slug:
@@ -38,6 +36,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur
         _, config_model = handler
         action_name = action_id.replace("_", " ").title()
         action_schema = json.loads(config_model.schema_json())
+        action_ui_schema = config_model.ui_schema()
         if issubclass(config_model, AuthActionConfiguration):
             action_type = ActionTypeEnum.AUTHENTICATION.value
         elif issubclass(config_model, PullActionConfiguration):
@@ -53,6 +52,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur
                 "value": action_id,
                 "description": f"{integration_type_name} {action_name} action",
                 "schema": action_schema,
+                "ui_schema": action_ui_schema,
                 "is_periodic_action": True if issubclass(config_model, PullActionConfiguration) else False,
             }
         )
@@ -70,6 +70,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur
             "value": f"{integration_type_slug}_webhook",
             "description": f"Webhook Integration with {integration_type_name}",
             "schema": json.loads(config_model.schema_json()),
+            "ui_schema": config_model.ui_schema(),
         }
 
     logger.info(f"Registering '{integration_type_slug}' with actions: '{actions}'")

app/services/tests/test_gundi_api.py

@@ -58,8 +58,8 @@ async def test_send_event_attachments_to_gundi(
     mocker.patch("app.services.gundi.GundiDataSenderClient", mock_gundi_sensors_client_class)
     mocker.patch("app.services.gundi._get_gundi_api_key", mock_get_gundi_api_key)
     attachments = [
-        b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00x\x00x\x00\x00\xff\xdb\x00C\x00\x02\x01\x01\x02',
-        b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x06\x01\x01\x00x\x00x\x01\x00\xff\xd5\x00C\x00\x98\x01\x01\x56'
+        ("file1.png", b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00x\x00x\x00\x00\xff\xdb\x00C\x00\x02\x01\x01\x02'),
+        ("file2.png", b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x06\x01\x01\x00x\x00x\x01\x00\xff\xd5\x00C\x00\x98\x01\x01\x56')
     ]
     response = await send_event_attachments_to_gundi(
         event_id="dummy-1234",