From 311c45a94fb2e61d5a4648e394741f7d01bae741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20H=2E=20Garc=C3=ADa?= Date: Mon, 15 Jul 2024 17:13:36 -0600 Subject: [PATCH 01/32] Add support for adding filenames in event attachments --- app/services/gundi.py | 7 +++++-- app/services/tests/test_gundi_api.py | 4 ++-- requirements-base.in | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/services/gundi.py b/app/services/gundi.py index 531d0ad..7990bac 100644 --- a/app/services/gundi.py +++ b/app/services/gundi.py @@ -52,11 +52,14 @@ async def send_events_to_gundi(events: List[dict], **kwargs) -> dict: @stamina.retry(on=httpx.HTTPError, wait_initial=1.0, wait_jitter=5.0, wait_max=32.0) -async def send_event_attachments_to_gundi(event_id: str, attachments: List[bytes], **kwargs) -> dict: +async def send_event_attachments_to_gundi(event_id: str, attachments: List[tuple], **kwargs) -> dict: """ Send Event Attachments to Gundi using the REST API v2 :param event_id: Created event in which the attachments are going to be linked - :param attachments: A list of attachments (in bytes) + :param attachments: A list of attachments (tuples with filename, file in bytes). Example: + filename = 'example.png' + file_in_bytes = open(filename, 'rb') + attachments = [(filename, file_in_bytes)] :param kwargs: integration_id: The UUID of the related integration :return: A dict with the response from the API """ diff --git a/app/services/tests/test_gundi_api.py b/app/services/tests/test_gundi_api.py index eae577f..1ed9ff5 100644 --- a/app/services/tests/test_gundi_api.py +++ b/app/services/tests/test_gundi_api.py @@ -58,8 +58,8 @@ async def test_send_event_attachments_to_gundi( mocker.patch("app.services.gundi.GundiDataSenderClient", mock_gundi_sensors_client_class) mocker.patch("app.services.gundi._get_gundi_api_key", mock_get_gundi_api_key) attachments = [ - b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00x\x00x\x00\x00\xff\xdb\x00C\x00\x02\x01\x01\x02', - b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x06\x01\x01\x00x\x00x\x01\x00\xff\xd5\x00C\x00\x98\x01\x01\x56' + ("file1.png", b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00x\x00x\x00\x00\xff\xdb\x00C\x00\x02\x01\x01\x02'), + ("file2.png", b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x06\x01\x01\x00x\x00x\x01\x00\xff\xd5\x00C\x00\x98\x01\x01\x56') ] response = await send_event_attachments_to_gundi( event_id="dummy-1234", diff --git a/requirements-base.in b/requirements-base.in index fd79276..9db68bd 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -5,7 +5,7 @@ pydantic~=1.10.15 fastapi~=0.103.2 uvicorn~=0.23.2 gundi-core~=1.5.0 -gundi-client-v2~=2.3.1 +gundi-client-v2~=2.3.2 stamina~=23.2.0 redis~=5.0.1 gcloud-aio-pubsub~=6.0.0 From 5bf2dec01013e6d5e93b92315ecfe0e75702ad4b Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Tue, 16 Jul 2024 10:18:08 -0300 Subject: [PATCH 02/32] add run unit test step --- .github/workflows/main.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 390f2b6..1fcdb07 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -4,6 +4,7 @@ on: branches: - main - 'release-**' + - 'gundi-3067' jobs: vars: @@ -33,6 +34,22 @@ jobs: workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} + run_unit_tests: + runs-on: ubuntu-latest + needs: [vars, build] + steps: + - name: Checkout branch + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.8' + - name: Install pip + run: python -m ensurepip --upgrade + - name: Install test runner + run: pip install -r requirements.in + - name: Run auth precondition test + run: pytest + deploy_dev: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/main') From bce260644389d7260e1f790a766a5b3f8d651d3b Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Tue, 16 Jul 2024 16:00:13 -0300 Subject: [PATCH 03/32] adding test dockerfile --- .github/workflows/main.yaml | 1 + docker/Dockerfile.test | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1fcdb07..5b015a9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -31,6 +31,7 @@ jobs: with: repository: ${{ needs.vars.outputs.repository }} tag: ${{ needs.vars.outputs.tag }} + dockerfile: docker/Dockerfile.test workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test new file mode 100644 index 0000000..d4775e9 --- /dev/null +++ b/docker/Dockerfile.test @@ -0,0 +1,14 @@ +FROM python:3.10-slim + +WORKDIR /code + +COPY requirements-base.in . +COPY requirements.in . +COPY requirements-dev.in . + +RUN python -m ensurepip --upgrade +RUN pip install pip-tools +RUN pip-compile --output-file=requirements.txt requirements-base.in requirements-dev.in requirements.in +COPY ./app app/ + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] From 257112b0052a69bd89becc6aae80eee6cf4472a2 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Tue, 16 Jul 2024 16:02:54 -0300 Subject: [PATCH 04/32] install pytest --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5b015a9..af9fb48 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -46,8 +46,8 @@ jobs: python-version: '3.8' - name: Install pip run: python -m ensurepip --upgrade - - name: Install test runner - run: pip install -r requirements.in + - name: Install pytest + run: pip install pytest - name: Run auth precondition test run: pytest From 6e3325921ba0ffca7a60ce319fd59a48548aebe5 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Tue, 16 Jul 2024 16:06:31 -0300 Subject: [PATCH 05/32] install base dependencies --- .github/workflows/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index af9fb48..686cd85 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -48,6 +48,8 @@ jobs: run: python -m ensurepip --upgrade - name: Install pytest run: pip install pytest + - name: Install dependencies + run: pip install -r requirements-base.in - name: Run auth precondition test run: pytest From 2296f102da1d696d00011af59c30a9797f4a26a2 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Wed, 17 Jul 2024 14:16:39 -0300 Subject: [PATCH 06/32] use python 3.10 --- .github/workflows/main.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 686cd85..08011a8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.10' - name: Install pip run: python -m ensurepip --upgrade - name: Install pytest @@ -56,7 +56,7 @@ jobs: deploy_dev: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/main') - needs: [vars, build] + needs: [run_unit_tests] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_dev }} @@ -69,7 +69,7 @@ jobs: deploy_stage: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/release') - needs: [vars, build] + needs: [run_unit_tests] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_stage }} @@ -82,7 +82,7 @@ jobs: deploy_prod: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/release') - needs: [vars, build, deploy_stage] + needs: [run_unit_tests, deploy_stage] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_prod }} From 11ab8582f9a18e9fa8239182e58417ff2f059c49 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Wed, 17 Jul 2024 15:24:57 -0300 Subject: [PATCH 07/32] adding dev dependencies --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 08011a8..9967c1e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,7 +49,7 @@ jobs: - name: Install pytest run: pip install pytest - name: Install dependencies - run: pip install -r requirements-base.in + run: pip install -r requirements-base.in -r requirements-dev.in - name: Run auth precondition test run: pytest From 8616e2189e2e58379cd81a2d16577c58b5044bc9 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Thu, 18 Jul 2024 11:14:52 -0300 Subject: [PATCH 08/32] Remove test branch --- .github/workflows/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9967c1e..77c51bb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -4,7 +4,6 @@ on: branches: - main - 'release-**' - - 'gundi-3067' jobs: vars: From d9c7b21ae8d9c377061bef1f439e1564992b1973 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Thu, 18 Jul 2024 13:21:21 -0300 Subject: [PATCH 09/32] removing test dockerfile --- .github/workflows/main.yaml | 1 - docker/Dockerfile.test | 14 -------------- 2 files changed, 15 deletions(-) delete mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 77c51bb..8e8dbc9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,7 +30,6 @@ jobs: with: repository: ${{ needs.vars.outputs.repository }} tag: ${{ needs.vars.outputs.tag }} - dockerfile: docker/Dockerfile.test workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test deleted file mode 100644 index d4775e9..0000000 --- a/docker/Dockerfile.test +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.10-slim - -WORKDIR /code - -COPY requirements-base.in . -COPY requirements.in . -COPY requirements-dev.in . - -RUN python -m ensurepip --upgrade -RUN pip install pip-tools -RUN pip-compile --output-file=requirements.txt requirements-base.in requirements-dev.in requirements.in -COPY ./app app/ - -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] From 9930d5a2d5e63ccae52312c0a70ff55bcf3b4d39 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Thu, 18 Jul 2024 14:12:14 -0300 Subject: [PATCH 10/32] fix unit test commands --- .github/workflows/main.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8e8dbc9..32f96d1 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -4,6 +4,7 @@ on: branches: - main - 'release-**' + - 'gundi-3067' jobs: vars: @@ -44,11 +45,13 @@ jobs: python-version: '3.10' - name: Install pip run: python -m ensurepip --upgrade - - name: Install pytest - run: pip install pytest + - name: Install pip tools + run: pip install pip-tools + - name: Install compile dependencies + run: pip-compile --output-file=requirements.txt requirements-base.in requirements-dev.in requirements.in - name: Install dependencies - run: pip install -r requirements-base.in -r requirements-dev.in - - name: Run auth precondition test + run: pip install -r requirements.txt + - name: Run unit tests run: pytest deploy_dev: From a09485f61689e133683123f5613f3ec2f9ddb925 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Thu, 18 Jul 2024 14:42:00 -0300 Subject: [PATCH 11/32] skip build step --- .github/workflows/main.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 32f96d1..4385a2a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -25,14 +25,14 @@ jobs: echo "tf_file_stage=environments/stage/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT echo "tf_file_prod=environments/prod/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT - build: - uses: PADAS/gundi-workflows/.github/workflows/build_docker.yml@v2 - needs: vars - with: - repository: ${{ needs.vars.outputs.repository }} - tag: ${{ needs.vars.outputs.tag }} - workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} - service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} + # build: + # uses: PADAS/gundi-workflows/.github/workflows/build_docker.yml@v2 + # needs: vars + # with: + # repository: ${{ needs.vars.outputs.repository }} + # tag: ${{ needs.vars.outputs.tag }} + # workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} + # service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} run_unit_tests: runs-on: ubuntu-latest From 09564d6fba4a5f22a4281b305da02253917cf09e Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Thu, 18 Jul 2024 14:42:59 -0300 Subject: [PATCH 12/32] removing dep step too --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4385a2a..e5b3b53 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -36,7 +36,7 @@ jobs: run_unit_tests: runs-on: ubuntu-latest - needs: [vars, build] + needs: [vars] steps: - name: Checkout branch uses: actions/checkout@v4 From ba3647b27335804c1aa194cb76e9c0d0177aa865 Mon Sep 17 00:00:00 2001 From: davidovichmarcos Date: Thu, 18 Jul 2024 14:48:56 -0300 Subject: [PATCH 13/32] fix --- .github/workflows/main.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e5b3b53..2e379df 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -4,7 +4,6 @@ on: branches: - main - 'release-**' - - 'gundi-3067' jobs: vars: @@ -24,15 +23,6 @@ jobs: echo "tf_file_dev=environments/dev/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT echo "tf_file_stage=environments/stage/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT echo "tf_file_prod=environments/prod/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT - - # build: - # uses: PADAS/gundi-workflows/.github/workflows/build_docker.yml@v2 - # needs: vars - # with: - # repository: ${{ needs.vars.outputs.repository }} - # tag: ${{ needs.vars.outputs.tag }} - # workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} - # service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} run_unit_tests: runs-on: ubuntu-latest @@ -53,11 +43,21 @@ jobs: run: pip install -r requirements.txt - name: Run unit tests run: pytest + + build: + uses: PADAS/gundi-workflows/.github/workflows/build_docker.yml@v2 + needs: [run_unit_tests, vars] + with: + repository: ${{ needs.vars.outputs.repository }} + tag: ${{ needs.vars.outputs.tag }} + workload_identity_provider: ${{ vars.GUNDI_INTEGRATIONS_WORKLOAD_IDENTITY_PROVIDER}} + service_account: ${{ vars.GUNDI_INTEGRATIONS_SERVICE_ACCOUNT }} + deploy_dev: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/main') - needs: [run_unit_tests] + needs: [build] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_dev }} @@ -70,7 +70,7 @@ jobs: deploy_stage: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/release') - needs: [run_unit_tests] + needs: [build] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_stage }} @@ -83,7 +83,7 @@ jobs: deploy_prod: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/release') - needs: [run_unit_tests, deploy_stage] + needs: [build, deploy_stage] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_prod }} From 4e72fd408756595864800433ada418b4a38e2b25 Mon Sep 17 00:00:00 2001 From: Daniel Fernandez Date: Thu, 18 Jul 2024 13:31:57 -0600 Subject: [PATCH 14/32] execute unit test on PR --- .github/workflows/_tests.yml | 23 +++++++++++++++++++++++ .github/workflows/main.yaml | 19 +------------------ .github/workflows/pr.yaml | 7 +++++++ 3 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/_tests.yml create mode 100644 .github/workflows/pr.yaml diff --git a/.github/workflows/_tests.yml b/.github/workflows/_tests.yml new file mode 100644 index 0000000..1818b76 --- /dev/null +++ b/.github/workflows/_tests.yml @@ -0,0 +1,23 @@ +name: Run tests +on: + workflow_call + +jobs: + run_unit_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install pip + run: python -m ensurepip --upgrade + - name: Install pip tools + run: pip install pip-tools + - name: Install compile dependencies + run: pip-compile --output-file=requirements.txt requirements-base.in requirements-dev.in requirements.in + - name: Install dependencies + run: pip install -r requirements.txt + - name: Run unit tests + run: pytest diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2e379df..a290f41 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -25,24 +25,7 @@ jobs: echo "tf_file_prod=environments/prod/integrations/${{ github.event.repository.name }}/terragrunt.hcl" >> $GITHUB_OUTPUT run_unit_tests: - runs-on: ubuntu-latest - needs: [vars] - steps: - - name: Checkout branch - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Install pip - run: python -m ensurepip --upgrade - - name: Install pip tools - run: pip install pip-tools - - name: Install compile dependencies - run: pip-compile --output-file=requirements.txt requirements-base.in requirements-dev.in requirements.in - - name: Install dependencies - run: pip install -r requirements.txt - - name: Run unit tests - run: pytest + uses: ./.github/workflows/_tests.yml build: uses: PADAS/gundi-workflows/.github/workflows/build_docker.yml@v2 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000..30a9c2e --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,7 @@ +name: Test integration +on: + pull_request + +jobs: + pr_unit_tests: + uses: ./.github/workflows/_tests.yml \ No newline at end of file From c714c28997dab837ced6a648634f04ab61be7571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20H=2E=20Garc=C3=ADa?= Date: Fri, 26 Jul 2024 21:14:16 -0600 Subject: [PATCH 15/32] Fix Dockerfile --- docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c7f3d2..7b6bdec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,8 @@ FROM python:3.10-slim +# This command is for webhooks support +RUN apt-get update && apt-get install -y autoconf automake libtool make python3-dev + WORKDIR /code COPY requirements.txt . From cc1d5aead9db873b87f52465e37a3dbfdd425bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20H=2E=20Garc=C3=ADa?= Date: Mon, 29 Jul 2024 15:07:44 -0600 Subject: [PATCH 16/32] Fix missing needed step in main.yaml` --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a290f41..3caa278 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,7 +40,7 @@ jobs: deploy_dev: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/main') - needs: [build] + needs: [vars, build] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_dev }} @@ -53,7 +53,7 @@ jobs: deploy_stage: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/release') - needs: [build] + needs: [vars, build] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_stage }} @@ -66,7 +66,7 @@ jobs: deploy_prod: uses: PADAS/gundi-workflows/.github/workflows/update_hcl.yml@v2 if: startsWith(github.ref, 'refs/heads/release') - needs: [build, deploy_stage] + needs: [vars, build, deploy_stage] with: git_repository: PADAS/gundi-integrations-v2-infra file_location: ${{ needs.vars.outputs.tf_file_prod }} From 40b95bea45e7c39953786cdc3bd6437ebf6465db Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 15 Oct 2024 08:18:45 -0300 Subject: [PATCH 17/32] Process pubsub messages in foreground by default --- app/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/settings/base.py b/app/settings/base.py index 400ff90..440a737 100644 --- a/app/settings/base.py +++ b/app/settings/base.py @@ -66,5 +66,5 @@ REGISTER_ON_START = env.bool("REGISTER_ON_START", False) INTEGRATION_TYPE_SLUG = env.str("INTEGRATION_TYPE_SLUG", None) # Define a string id here e.g. "my_tracker" INTEGRATION_SERVICE_URL = env.str("INTEGRATION_SERVICE_URL", None) # Define a string id here e.g. "my_tracker" -PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND = env.bool("PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND", True) +PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND = env.bool("PROCESS_PUBSUB_MESSAGES_IN_BACKGROUND", False) PROCESS_WEBHOOKS_IN_BACKGROUND = env.bool("PROCESS_WEBHOOKS_IN_BACKGROUND", True) From 53d707c85016846f90dedb0e24fa2b34368ac9e1 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 09:45:42 -0300 Subject: [PATCH 18/32] Support ui schema in action config models --- app/actions/core.py | 9 +- app/services/self_registration.py | 4 +- app/services/utils.py | 214 +++++++++++++++++++++++++++++- 3 files changed, 220 insertions(+), 7 deletions(-) diff --git a/app/actions/core.py b/app/actions/core.py index f646b4d..7a9d115 100644 --- a/app/actions/core.py +++ b/app/actions/core.py @@ -1,9 +1,16 @@ import importlib import inspect +from typing import Optional + from pydantic import BaseModel +from app.services.utils import UISchemaModelMixin + + +class ActionConfiguration(UISchemaModelMixin, BaseModel): + pass -class ActionConfiguration(BaseModel): +class ExecutableActionMixin: pass diff --git a/app/services/self_registration.py b/app/services/self_registration.py index 2ddce42..8596c2e 100644 --- a/app/services/self_registration.py +++ b/app/services/self_registration.py @@ -14,8 +14,6 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_url=None): - #from ..webhooks.configurations import LiquidTechPayload - #print(GenericJsonTransformConfig.schema_json()) # Prepare the integration name and value integration_type_slug = type_slug or INTEGRATION_TYPE_SLUG if not integration_type_slug: @@ -38,6 +36,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur _, config_model = handler action_name = action_id.replace("_", " ").title() action_schema = json.loads(config_model.schema_json()) + action_ui_schema = config_model.ui_schema() if issubclass(config_model, AuthActionConfiguration): action_type = ActionTypeEnum.AUTHENTICATION.value elif issubclass(config_model, PullActionConfiguration): @@ -53,6 +52,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur "value": action_id, "description": f"{integration_type_name} {action_name} action", "schema": action_schema, + "ui_schema": action_ui_schema, "is_periodic_action": True if issubclass(config_model, PullActionConfiguration) else False, } ) diff --git a/app/services/utils.py b/app/services/utils.py index 673cd66..d83b901 100644 --- a/app/services/utils.py +++ b/app/services/utils.py @@ -1,9 +1,9 @@ import struct -from typing import Annotated, Union +from typing import Annotated import typing -from pydantic import create_model -from pydantic.fields import Field - +from pydantic import create_model, BaseModel +from pydantic.fields import Field, FieldInfo, Undefined, NoArgAnyCallable +from typing import Any, Dict, Optional, Union, List def find_config_for_action(configurations, action_id): @@ -167,3 +167,209 @@ def _make_field(self, factory, field, alias) -> None: ... ) + +class GlobalUISchemaOptions(BaseModel): + order: Optional[List[str]] + addable: Optional[bool] = Field(default=True) + copyable: Optional[bool] = Field(default=False) + orderable: Optional[bool] = Field(default=True) + removable: Optional[bool] = Field(default=True) + label: Optional[bool] = Field(default=True) + duplicateKeySuffixSeparator: Optional[str] = Field(default='-') + + +class UIOptions(GlobalUISchemaOptions): + classNames: Optional[str] + style: Optional[Dict[str, Any]] # Assuming style is a dictionary of CSS properties + title: Optional[str] + description: Optional[str] + placeholder: Optional[str] + help: Optional[str] + autofocus: Optional[bool] + autocomplete: Optional[str] # Type of HTMLInputElement['autocomplete'] + disabled: Optional[bool] + emptyValue: Optional[Any] + enumDisabled: Optional[Union[List[Union[str, int, bool]], None]] # List of disabled enum options + hideError: Optional[bool] + readonly: Optional[bool] + filePreview: Optional[bool] + inline: Optional[bool] + inputType: Optional[str] + rows: Optional[int] + submitButtonOptions: Optional[Dict[str, Any]] # Assuming UISchemaSubmitButtonOptions is a dict + widget: Optional[Union[str, Any]] # Either a widget implementation or its name + enumNames: Optional[List[str]] # List of labels for enum values + + +class FieldInfoWithUIOptions(FieldInfo): + + def __init__(self, *args, **kwargs): + """ + Extends the Pydantic Field class to support ui:schema generation + :param kwargs: ui_options: UIOptions + """ + self.ui_options = kwargs.pop("ui_options", None) + super().__init__(*args, **kwargs) + + def ui_schema(self, *args, **kwargs): + """Generates a UI schema from model field ui_schema""" + if not self.ui_options: + return {} + ui_schema = {} + ui_options = self.ui_options.__fields__ + for field_name, model_field in ui_options.items(): + if value := getattr(self.ui_options, field_name, model_field.default): + ui_schema[f"ui:{field_name}"] = value + return ui_schema + + +def FieldWithUIOptions( + default: Any = Undefined, + *, + default_factory: Optional[NoArgAnyCallable] = None, + alias: Optional[str] = None, + title: Optional[str] = None, + description: Optional[str] = None, + exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny', Any]] = None, + include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny', Any]] = None, + const: Optional[bool] = None, + gt: Optional[float] = None, + ge: Optional[float] = None, + lt: Optional[float] = None, + le: Optional[float] = None, + multiple_of: Optional[float] = None, + allow_inf_nan: Optional[bool] = None, + max_digits: Optional[int] = None, + decimal_places: Optional[int] = None, + min_items: Optional[int] = None, + max_items: Optional[int] = None, + unique_items: Optional[bool] = None, + min_length: Optional[int] = None, + max_length: Optional[int] = None, + allow_mutation: bool = True, + regex: Optional[str] = None, + discriminator: Optional[str] = None, + repr: bool = True, + ui_options: UIOptions = None, + **extra: Any, +) -> FieldInfoWithUIOptions: + """ + Used to provide extra information about a field, either for the model schema or complex validation. Some arguments + apply only to number fields (``int``, ``float``, ``Decimal``) and some apply only to ``str``. + + :param default: since this is replacing the field’s default, its first argument is used + to set the default, use ellipsis (``...``) to indicate the field is required + :param default_factory: callable that will be called when a default value is needed for this field + If both `default` and `default_factory` are set, an error is raised. + :param alias: the public name of the field + :param title: can be any string, used in the schema + :param description: can be any string, used in the schema + :param exclude: exclude this field while dumping. + Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method. + :param include: include this field while dumping. + Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method. + :param const: this field is required and *must* take it's default value + :param gt: only applies to numbers, requires the field to be "greater than". The schema + will have an ``exclusiveMinimum`` validation keyword + :param ge: only applies to numbers, requires the field to be "greater than or equal to". The + schema will have a ``minimum`` validation keyword + :param lt: only applies to numbers, requires the field to be "less than". The schema + will have an ``exclusiveMaximum`` validation keyword + :param le: only applies to numbers, requires the field to be "less than or equal to". The + schema will have a ``maximum`` validation keyword + :param multiple_of: only applies to numbers, requires the field to be "a multiple of". The + schema will have a ``multipleOf`` validation keyword + :param allow_inf_nan: only applies to numbers, allows the field to be NaN or infinity (+inf or -inf), + which is a valid Python float. Default True, set to False for compatibility with JSON. + :param max_digits: only applies to Decimals, requires the field to have a maximum number + of digits within the decimal. It does not include a zero before the decimal point or trailing decimal zeroes. + :param decimal_places: only applies to Decimals, requires the field to have at most a number of decimal places + allowed. It does not include trailing decimal zeroes. + :param min_items: only applies to lists, requires the field to have a minimum number of + elements. The schema will have a ``minItems`` validation keyword + :param max_items: only applies to lists, requires the field to have a maximum number of + elements. The schema will have a ``maxItems`` validation keyword + :param unique_items: only applies to lists, requires the field not to have duplicated + elements. The schema will have a ``uniqueItems`` validation keyword + :param min_length: only applies to strings, requires the field to have a minimum length. The + schema will have a ``minLength`` validation keyword + :param max_length: only applies to strings, requires the field to have a maximum length. The + schema will have a ``maxLength`` validation keyword + :param allow_mutation: a boolean which defaults to True. When False, the field raises a TypeError if the field is + assigned on an instance. The BaseModel Config must set validate_assignment to True + :param regex: only applies to strings, requires the field match against a regular expression + pattern string. The schema will have a ``pattern`` validation keyword + :param discriminator: only useful with a (discriminated a.k.a. tagged) `Union` of sub models with a common field. + The `discriminator` is the name of this common field to shorten validation and improve generated schema + :param repr: show this field in the representation + :param ui_options: UIOptions instance used to set ui properties for the ui schema + :param **extra: any additional keyword arguments will be added as is to the schema + """ + field_info = FieldInfoWithUIOptions( + default, + default_factory=default_factory, + alias=alias, + title=title, + description=description, + exclude=exclude, + include=include, + const=const, + gt=gt, + ge=ge, + lt=lt, + le=le, + multiple_of=multiple_of, + allow_inf_nan=allow_inf_nan, + max_digits=max_digits, + decimal_places=decimal_places, + min_items=min_items, + max_items=max_items, + unique_items=unique_items, + min_length=min_length, + max_length=max_length, + allow_mutation=allow_mutation, + regex=regex, + discriminator=discriminator, + repr=repr, + ui_options=ui_options, + **extra, + ) + field_info._validate() + return field_info + + +class UISchemaModelMixin: + + @classmethod + def ui_schema(cls, *args, **kwargs): + """Generates a UI schema from model""" + ui_schema = {} + # Iterate through the fields and generate UI schema + for field_name, model_field in cls.__fields__.items(): + if getattr(model_field.field_info, "ui_options", None): + ui_schema[field_name] = model_field.field_info.ui_schema() + # Include global options + if global_options := cls.__fields__.get('ui_global_options'): + if getattr(global_options, "type_", None) == GlobalUISchemaOptions: + model = global_options.default + for field_name, model_field in model.__fields__.items(): + if value := getattr(model, field_name, model_field.default): + ui_schema[f"ui:{field_name}"] = value + return ui_schema + + + @classmethod + def schema(cls, **kwargs): + # Call the parent schema method to get the original schema + json_schema_dict = super().schema(**kwargs) + + # Remove ui schema fields from the properties and definitions + properties = json_schema_dict.get('properties', {}) + for field in ["ui_options", "ui_global_options"]: + properties.pop(field, None) + json_schema_dict['properties'] = properties + definitions = json_schema_dict.get('definitions', {}) + for field in ["UIOptions", "GlobalUISchemaOptions"]: + definitions.pop(field, None) + json_schema_dict['definitions'] = definitions + return json_schema_dict From 6375455b73e286616d457494299e0a6f14755a50 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 09:57:04 -0300 Subject: [PATCH 19/32] Minor improvements --- app/actions/core.py | 4 ---- app/services/utils.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/actions/core.py b/app/actions/core.py index 7a9d115..a7dbe09 100644 --- a/app/actions/core.py +++ b/app/actions/core.py @@ -10,10 +10,6 @@ class ActionConfiguration(UISchemaModelMixin, BaseModel): pass -class ExecutableActionMixin: - pass - - class PullActionConfiguration(ActionConfiguration): pass diff --git a/app/services/utils.py b/app/services/utils.py index d83b901..c6241c6 100644 --- a/app/services/utils.py +++ b/app/services/utils.py @@ -212,7 +212,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def ui_schema(self, *args, **kwargs): - """Generates a UI schema from model field ui_schema""" + """Generates a UI schema from model field ui_options""" if not self.ui_options: return {} ui_schema = {} From 2a4c9a0951a1b76926717f0ff827e71250b993c2 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 11:44:38 -0300 Subject: [PATCH 20/32] Fix unit tests for self-registration --- app/services/tests/test_self_registration.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index 2e18579..1ed7325 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -33,9 +33,11 @@ async def test_register_integration_with_slug_setting( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } @@ -95,9 +97,11 @@ async def test_register_integration_with_slug_arg( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } @@ -162,9 +166,11 @@ async def test_register_integration_with_service_url_arg( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } @@ -229,9 +235,11 @@ async def test_register_integration_with_service_url_setting( 'type': 'integer' } }, + 'definitions': {}, 'title': 'MockPullActionConfiguration', 'type': 'object' }, + "ui_schema": {}, 'type': 'pull', 'value': 'pull_observations' } From 6bc24d6acfa93bf091a664e2cee87a28cc55a9e7 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 12:08:58 -0300 Subject: [PATCH 21/32] Test coverage for ui schemas --- app/services/tests/test_self_registration.py | 246 ++++++++++++++----- 1 file changed, 185 insertions(+), 61 deletions(-) diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index 1ed7325..9f3a241 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -22,24 +22,55 @@ async def test_register_integration_with_slug_setting( "description": f"Default type for integrations with X Tracker", "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" + }, + "is_periodic_action": True } ], "webhook": { @@ -86,24 +117,55 @@ async def test_register_integration_with_slug_arg( "description": f"Default type for integrations with X Tracker", "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} + }, + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "is_periodic_action": True } ], "webhook": { @@ -155,24 +217,55 @@ async def test_register_integration_with_service_url_arg( 'service_url': service_url, "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" + }, + "is_periodic_action": True } ], "webhook": { @@ -213,7 +306,7 @@ async def test_register_integration_with_service_url_setting( mocker.patch("app.services.self_registration.action_handlers", mock_action_handlers) mocker.patch("app.services.self_registration.get_webhook_handler", mock_get_webhook_handler_for_fixed_json_payload) - await register_integration_in_gundi(gundi_client=mock_gundi_client_v2,) + await register_integration_in_gundi(gundi_client=mock_gundi_client_v2, ) assert mock_gundi_client_v2.register_integration_type.called mock_gundi_client_v2.register_integration_type.assert_called_with( @@ -224,24 +317,55 @@ async def test_register_integration_with_service_url_setting( 'service_url': service_url, "actions": [ { - 'description': 'X Tracker Pull Observations action', - 'is_periodic_action': True, - 'name': 'Pull Observations', - 'schema': { - 'properties': { - 'lookback_days': { - 'default': 10, - 'title': 'Lookback Days', - 'type': 'integer' + "type": "pull", + "name": "Pull Observations", + "value": "pull_observations", + "description": "X Tracker Pull Observations action", + "schema": { + "title": "MockPullActionConfiguration", + "type": "object", + "properties": { + "lookback_days": { + "title": "Data lookback days", + "description": "Number of days to look back for data.", + "default": 30, + "minimum": 1, "maximum": 30, + "type": "integer" + }, + "force_fetch": { + "title": "Force fetch", + "description": "Force fetch even if in a quiet period.", + "default": False, + "type": "boolean" } }, - 'definitions': {}, - 'title': 'MockPullActionConfiguration', - 'type': 'object' + "definitions": {} + }, + "ui_schema": { + "lookback_days": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "range" + }, + "force_fetch": { + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" + }, + "ui:order": ["lookback_days", "force_fetch"], + "ui:addable": True, + "ui:orderable": True, + "ui:removable": True, + "ui:label": True, + "ui:duplicateKeySuffixSeparator": "-" }, - "ui_schema": {}, - 'type': 'pull', - 'value': 'pull_observations' + "is_periodic_action": True } ], "webhook": { From ee2d0d19affa3ab364463c201f64f5fcf06013df Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 12:35:02 -0300 Subject: [PATCH 22/32] Test coverage for ui schemas --- app/conftest.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index 8f62237..2e69aec 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -1,15 +1,13 @@ import asyncio import datetime import json - import pydantic import pytest from unittest.mock import MagicMock from app import settings from gcloud.aio import pubsub -from gundi_core.schemas.v2 import Integration, IntegrationActionConfiguration, IntegrationActionSummary +from gundi_core.schemas.v2 import Integration from gundi_core.events import ( - SystemEventBaseModel, IntegrationActionCustomLog, CustomActivityLog, IntegrationActionStarted, @@ -28,8 +26,8 @@ CustomWebhookLog, LogLevel ) - from app.actions import PullActionConfiguration +from app.services.utils import GlobalUISchemaOptions, FieldWithUIOptions, UIOptions from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload @@ -898,7 +896,30 @@ def mock_publish_event(gcp_pubsub_publish_response): class MockPullActionConfiguration(PullActionConfiguration): - lookback_days: int = 10 + lookback_days: int = FieldWithUIOptions( + 30, + le=30, + ge=1, + title="Data lookback days", + description="Number of days to look back for data.", + ui_options=UIOptions( + widget="range", + ) + ) + force_fetch: bool = FieldWithUIOptions( + False, + title="Force fetch", + description="Force fetch even if in a quiet period.", + ui_options=UIOptions( + widget="select", + ) + ) + ui_global_options = GlobalUISchemaOptions( + order=[ + "lookback_days", + "force_fetch", + ], + ) @pytest.fixture From 913760f82471794ed5e6f5ffcbfef6760693d1a3 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 15:08:10 -0300 Subject: [PATCH 23/32] Remove unecessary default properties from ui schema --- app/services/tests/test_self_registration.py | 61 +------------------- app/services/utils.py | 12 ++-- 2 files changed, 7 insertions(+), 66 deletions(-) diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index 9f3a241..a090865 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -48,27 +48,12 @@ async def test_register_integration_with_slug_setting( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } @@ -143,27 +128,13 @@ async def test_register_integration_with_slug_arg( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", + "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } @@ -243,27 +214,12 @@ async def test_register_integration_with_service_url_arg( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } @@ -343,27 +299,12 @@ async def test_register_integration_with_service_url_setting( }, "ui_schema": { "lookback_days": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "range" }, "force_fetch": { - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-", "ui:widget": "select" }, "ui:order": ["lookback_days", "force_fetch"], - "ui:addable": True, - "ui:orderable": True, - "ui:removable": True, - "ui:label": True, - "ui:duplicateKeySuffixSeparator": "-" }, "is_periodic_action": True } diff --git a/app/services/utils.py b/app/services/utils.py index c6241c6..da4a1b5 100644 --- a/app/services/utils.py +++ b/app/services/utils.py @@ -170,12 +170,12 @@ def _make_field(self, factory, field, alias) -> None: class GlobalUISchemaOptions(BaseModel): order: Optional[List[str]] - addable: Optional[bool] = Field(default=True) - copyable: Optional[bool] = Field(default=False) - orderable: Optional[bool] = Field(default=True) - removable: Optional[bool] = Field(default=True) - label: Optional[bool] = Field(default=True) - duplicateKeySuffixSeparator: Optional[str] = Field(default='-') + addable: Optional[bool] + copyable: Optional[bool] + orderable: Optional[bool] + removable: Optional[bool] + label: Optional[bool] + duplicateKeySuffixSeparator: Optional[str] class UIOptions(GlobalUISchemaOptions): From bd2761ce0dac132cae9a4fa34fc9d4c1786debf6 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 15:31:47 -0300 Subject: [PATCH 24/32] Support ui schemas in webhook configs --- app/services/self_registration.py | 1 + app/webhooks/core.py | 36 +++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/app/services/self_registration.py b/app/services/self_registration.py index 8596c2e..88e68a9 100644 --- a/app/services/self_registration.py +++ b/app/services/self_registration.py @@ -70,6 +70,7 @@ async def register_integration_in_gundi(gundi_client, type_slug=None, service_ur "value": f"{integration_type_slug}_webhook", "description": f"Webhook Integration with {integration_type_name}", "schema": json.loads(config_model.schema_json()), + "ui_schema": config_model.ui_schema(), } logger.info(f"Registering '{integration_type_slug}' with actions: '{actions}'") diff --git a/app/webhooks/core.py b/app/webhooks/core.py index 855761c..25653c0 100644 --- a/app/webhooks/core.py +++ b/app/webhooks/core.py @@ -3,12 +3,11 @@ import json from typing import Optional, Union from pydantic import BaseModel -from pydantic.fields import Field from fastapi.encoders import jsonable_encoder -from app.services.utils import StructHexString +from app.services.utils import StructHexString, UISchemaModelMixin, FieldWithUIOptions, UIOptions -class WebhookConfiguration(BaseModel): +class WebhookConfiguration(UISchemaModelMixin, BaseModel): class Config: extra = "allow" @@ -19,26 +18,45 @@ class HexStringConfig(WebhookConfiguration): class DynamicSchemaConfig(WebhookConfiguration): - json_schema: dict + json_schema: dict = FieldWithUIOptions( + default={}, + description="JSON Schema to validate the data.", + ui_options=UIOptions( + widget="textarea", # ToDo: Use a better (custom) widget to render the JSON schema + ) + ) -class JQTransformConfig(BaseModel): - jq_filter: str = Field( +class JQTransformConfig(UISchemaModelMixin, BaseModel): + jq_filter: str = FieldWithUIOptions( default=".", description="JQ filter to transform JSON data.", - example=". | map(select(.isActive))" + example=". | map(select(.isActive))", + ui_options=UIOptions( + widget="textarea", # ToDo: Use a better (custom) widget to render the JQ filter + ) ) class GenericJsonTransformConfig(JQTransformConfig, DynamicSchemaConfig): - output_type: str = Field(..., description="Output type for the transformed data: 'obv' or 'event'") + output_type: str = FieldWithUIOptions( + ..., + description="Output type for the transformed data: 'obv' or 'event'", + ui_options=UIOptions( + widget="select", + options=[ + {"label": "Observations", "value": "obv"}, + {"label": "Events", "value": "event"}, + ] + ) + ) class GenericJsonTransformWithHexStrConfig(HexStringConfig, GenericJsonTransformConfig): pass -class WebhookPayload(BaseModel): +class WebhookPayload(UISchemaModelMixin, BaseModel): class Config: extra = "allow" From 1a4d90cd4ee3b53909e31564474dc7dd7523b264 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 16:07:26 -0300 Subject: [PATCH 25/32] Change default widget for output type used in generic integrations --- app/webhooks/core.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/webhooks/core.py b/app/webhooks/core.py index 25653c0..8b1288d 100644 --- a/app/webhooks/core.py +++ b/app/webhooks/core.py @@ -43,11 +43,7 @@ class GenericJsonTransformConfig(JQTransformConfig, DynamicSchemaConfig): ..., description="Output type for the transformed data: 'obv' or 'event'", ui_options=UIOptions( - widget="select", - options=[ - {"label": "Observations", "value": "obv"}, - {"label": "Events", "value": "event"}, - ] + widget="text", # ToDo: Use a select or a better widget to render the output type ) ) From 1ea4aba09d12e5d41a3c565945bc35d07beda189 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 16:56:37 -0300 Subject: [PATCH 26/32] Add test coverage for webhooks with ui schema --- app/conftest.py | 39 ++++++++++++++++++-- app/services/tests/test_self_registration.py | 28 ++++++++++++-- app/webhooks/core.py | 2 +- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index 2e69aec..07d7733 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -28,7 +28,7 @@ ) from app.actions import PullActionConfiguration from app.services.utils import GlobalUISchemaOptions, FieldWithUIOptions, UIOptions -from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload +from app.webhooks import GenericJsonTransformConfig, GenericJsonPayload, WebhookPayload, WebhookConfiguration class AsyncMock(MagicMock): @@ -137,6 +137,14 @@ def integration_v2_with_webhook(): "allowed_devices_list": {"title": "Allowed Devices List", "type": "array", "items": {}}, "deduplication_enabled": {"title": "Deduplication Enabled", "type": "boolean"}}, "required": ["allowed_devices_list", "deduplication_enabled"] + }, + "ui_schema": { + "allowed_devices_list": { + "ui:widget": "select" + }, + "deduplication_enabled": { + "ui:widget": "radio" + } } } }, @@ -216,6 +224,17 @@ def integration_v2_with_webhook_generic(): "description": "Output type for the transformed data: 'obv' or 'event'" } } + }, + "ui_schema": { + "jq_filter": { + "ui:widget": "textarea" + }, + "json_schema": { + "ui:widget": "textarea" + }, + "output_type": { + "ui:widget": "text" + } } } }, @@ -1193,9 +1212,21 @@ class MockWebhookPayloadModel(WebhookPayload): lon: float -class MockWebhookConfigModel(pydantic.BaseModel): - allowed_devices_list: list - deduplication_enabled: bool +class MockWebhookConfigModel(WebhookConfiguration): + allowed_devices_list: list = FieldWithUIOptions( + ..., + title="Allowed Devices List", + ui_options=UIOptions( + widget="list", + ) + ) + deduplication_enabled: bool = FieldWithUIOptions( + ..., + title="Deduplication Enabled", + ui_options=UIOptions( + widget="radio", + ) + ) @pytest.fixture diff --git a/app/services/tests/test_self_registration.py b/app/services/tests/test_self_registration.py index a090865..302d2a9 100644 --- a/app/services/tests/test_self_registration.py +++ b/app/services/tests/test_self_registration.py @@ -76,11 +76,16 @@ async def test_register_integration_with_slug_setting( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) @@ -157,11 +162,16 @@ async def test_register_integration_with_slug_arg( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) @@ -242,11 +252,16 @@ async def test_register_integration_with_service_url_arg( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) @@ -327,11 +342,16 @@ async def test_register_integration_with_service_url_setting( "type": "boolean" } }, + "definitions": {}, "required": [ "allowed_devices_list", "deduplication_enabled" ] - } + }, + "ui_schema": { + "allowed_devices_list": {"ui:widget": "list"}, + "deduplication_enabled": {"ui:widget": "radio"} + }, } } ) diff --git a/app/webhooks/core.py b/app/webhooks/core.py index 8b1288d..da32b79 100644 --- a/app/webhooks/core.py +++ b/app/webhooks/core.py @@ -52,7 +52,7 @@ class GenericJsonTransformWithHexStrConfig(HexStringConfig, GenericJsonTransform pass -class WebhookPayload(UISchemaModelMixin, BaseModel): +class WebhookPayload(BaseModel): class Config: extra = "allow" From b6fda511e42f8a35c73bdd984364993a34a809c0 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Fri, 25 Oct 2024 17:54:31 -0300 Subject: [PATCH 27/32] Add usage examples for ui schemas in the readme --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/README.md b/README.md index 555d834..8f95e02 100644 --- a/README.md +++ b/README.md @@ -403,3 +403,57 @@ Sample configuration in Gundi: """ ``` Notice: This can also be combined with Dynamic Schema and JSON Transformations. In that case the hex string will be parsed first, adn then the JQ filter can be applied to the extracted data. + +### Custom UI for configurations (ui schema) +It's possible to customize how the forms for configurations are displayed in the Gundi portal. +To do that, use `FieldWithUIOptions` in your models. The `UIOptions` and `GlobalUISchemaOptions` will allow you to customize the appearance of the fields in the portal by setting any of the ["ui schema"](https://rjsf-team.github.io/react-jsonschema-form/docs/api-reference/uiSchema) supported options. + +```python +# Example +import pydantic +from app.services.utils import FieldWithUIOptions, GlobalUISchemaOptions, UIOptions +from .core import AuthActionConfiguration, PullActionConfiguration + + +class AuthenticateConfig(AuthActionConfiguration): + email: str # This will be rendered with default widget and settings + password: pydantic.SecretStr = FieldWithUIOptions( + ..., + format="password", + title="Password", + description="Password for the Global Forest Watch account.", + ui_options=UIOptions( + widget="password", # This will be rendered as a password input hiding the input + ) + ) + ui_global_options = GlobalUISchemaOptions( + order=["email", "password"], # This will set the order of the fields in the form + ) + + +class MyPullActionConfiguration(PullActionConfiguration): + lookback_days: int = FieldWithUIOptions( + 10, + le=30, + ge=1, + title="Data lookback days", + description="Number of days to look back for data.", + ui_options=UIOptions( + widget="range", # This will be rendered ad a range slider + ) + ) + force_fetch: bool = FieldWithUIOptions( + False, + title="Force fetch", + description="Force fetch even if in a quiet period.", + ui_options=UIOptions( + widget="radio", # This will be rendered as a radio button + ) + ) + ui_global_options = GlobalUISchemaOptions( + order=[ + "lookback_days", + "force_fetch", + ], + ) +``` From 3d68ea087b106b5b9a06544329906306bc94b060 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 29 Oct 2024 13:40:45 -0300 Subject: [PATCH 28/32] Update dependencies --- requirements-base.in | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements-base.in b/requirements-base.in index 9db68bd..e22173e 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -4,10 +4,11 @@ environs~=9.5.0 pydantic~=1.10.15 fastapi~=0.103.2 uvicorn~=0.23.2 -gundi-core~=1.5.0 -gundi-client-v2~=2.3.2 +gundi-core~=1.7.0 +gundi-client-v2~=2.3.8 stamina~=23.2.0 redis~=5.0.1 gcloud-aio-pubsub~=6.0.0 click~=8.1.7 -pyjq~=2.6.0 +#pyjq~=2.6.0 +python-json-logger~=2.0.7 From 633621e7029640b69c84185708cc2c2280df0645 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 29 Oct 2024 13:42:57 -0300 Subject: [PATCH 29/32] Update dependencies --- requirements-base.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-base.in b/requirements-base.in index e22173e..cb9441e 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -10,5 +10,5 @@ stamina~=23.2.0 redis~=5.0.1 gcloud-aio-pubsub~=6.0.0 click~=8.1.7 -#pyjq~=2.6.0 +pyjq~=2.6.0 python-json-logger~=2.0.7 From 8d9dca833fdeb1ad7e575b39bb082e17a4e7f304 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 5 Nov 2024 16:05:59 -0300 Subject: [PATCH 30/32] Update api mocks for integration status --- app/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index 07d7733..c48044c 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -105,9 +105,9 @@ def integration_v2(): 'value': 'auth'}, 'data': {'token': 'testtoken2a97022f21732461ee103a08fac8a35'}}], 'additional': {}, 'default_route': {'id': '5abf3845-7c9f-478a-bc0f-b24d87038c4b', 'name': 'Gundi X Provider - Default Route'}, - 'status': {'id': 'mockid-b16a-4dbd-ad32-197c58aeef59', 'is_healthy': True, - 'details': 'Last observation has been delivered with success.', - 'observation_delivered_24hrs': 50231, 'last_observation_delivered_at': '2023-03-31T11:20:00+0200'}} + 'status': 'healthy', + 'status_details': '', + } ) From a33d42f6dd6118698fe43e82cee363008432d706 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Tue, 5 Nov 2024 16:09:12 -0300 Subject: [PATCH 31/32] Update api mocks for integration status --- app/conftest.py | 18 ++++-------------- requirements-base.in | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/app/conftest.py b/app/conftest.py index c48044c..ae20973 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -169,13 +169,8 @@ def integration_v2_with_webhook(): }, "additional": {}, "default_route": None, - "status": { - "id": "mockid-b16a-4dbd-ad32-197c58aeef59", - "is_healthy": True, - "details": "Last observation has been delivered with success.", - "observation_delivered_24hrs": 50231, - "last_observation_delivered_at": "2023-03-31T11:20:00+0200" - } + "status": "healthy", + "status_details": "", } ) @@ -472,13 +467,8 @@ def integration_v2_with_webhook_generic(): }, "additional": {}, "default_route": None, - "status": { - "id": "mockid-b16a-4dbd-ad32-197c58aeef59", - "is_healthy": True, - "details": "Last observation has been delivered with success.", - "observation_delivered_24hrs": 50231, - "last_observation_delivered_at": "2023-03-31T11:20:00+0200" - } + "status": "healthy", + "status_details": "", } ) diff --git a/requirements-base.in b/requirements-base.in index cb9441e..7c4e316 100644 --- a/requirements-base.in +++ b/requirements-base.in @@ -4,7 +4,7 @@ environs~=9.5.0 pydantic~=1.10.15 fastapi~=0.103.2 uvicorn~=0.23.2 -gundi-core~=1.7.0 +gundi-core~=1.7.1 gundi-client-v2~=2.3.8 stamina~=23.2.0 redis~=5.0.1 From 33a85833fb70a6d7b5d38f440b1e41267c7697b4 Mon Sep 17 00:00:00 2001 From: Mariano Martinez Grasso Date: Wed, 6 Nov 2024 18:06:21 -0300 Subject: [PATCH 32/32] re-generate requirements --- requirements.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f131da0..f49bbcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,9 +51,9 @@ gcloud-aio-auth==5.3.1 # via gcloud-aio-pubsub gcloud-aio-pubsub==6.0.1 # via -r requirements-base.in -gundi-client-v2==2.2.0 +gundi-client-v2==2.3.8 # via -r requirements-base.in -gundi-core==1.3.0 +gundi-core==1.7.1 # via # -r requirements-base.in # gundi-client-v2 @@ -90,10 +90,14 @@ pydantic==1.10.15 # fastapi # gundi-client-v2 # gundi-core +pyjq==2.6.0 + # via -r requirements-base.in pyjwt==2.8.0 # via gcloud-aio-auth python-dotenv==1.0.1 # via environs +python-json-logger==2.0.7 + # via -r requirements-base.in redis==5.0.4 # via -r requirements-base.in respx==0.20.2