Release: v7.0.9 [API] v0.1.7 [SDK]

.env.example

@@ -8,12 +8,11 @@ RESOURCE_PREFIX=rapid
 DOMAIN_NAME=example.com
 COGNITO_USER_POOL_ID=11111111
 LAYERS=raw,layer
-
+CUSTOM_USER_NAME_REGEX="regex_expression"
 # SDK Specific
 RAPID_CLIENT_ID=
 RAPID_CLIENT_SECRET=
 RAPID_URL=
-
 # UI Specific
 NEXT_PUBLIC_API_URL=
 NEXT_PUBLIC_API_URL_PROXY=

.github/.github.env

@@ -4,3 +4,4 @@ ALLOWED_EMAIL_DOMAINS=example1.com,example2.com
 LAYERS=raw,layer
 DOMAIN_NAME=example.com
 DATA_BUCKET=the-bucket
+CUSTOM_USER_NAME_REGEX=[a-zA-Z][a-zA-Z0-9@._-]{2,127}

.github/workflows/dev.yml

@@ -90,10 +90,12 @@ jobs:
       - name: SDK Test
         run: make sdk-test
 
-      # TODO: Add back in
-      # - name: SDK Test Deploy
-      #   run: make sdk-release-test
+      - name: Set env variable
+        run: echo "TEST_SDK_VERSION=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
 
+      - name: SDK Test Deploy
+        run: make sdk-release-test
+
   ui-dev:
     needs:
       - setup

.github/workflows/release.yml → .github/workflows/release_api.yml

@@ -6,6 +6,7 @@ on:
 
 jobs:
   setup:
+    if: github.event.release.name == 'api release'
     runs-on: self-hosted
     steps:
       - name: Checkout
@@ -34,34 +35,6 @@ jobs:
       - name: API Tag and Upload Release Image
         run: make api-tag-and-upload-release-image
 
-  sdk-release:
-    needs:
-      - setup
-    runs-on: self-hosted
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Populate .env with additional vars
-        run: |
-          echo TWINE_USERNAME=${{ secrets.TWINE_USERNAME }} >> .env
-          echo TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} >> .env
-          echo TWINE_NON_INTERACTIVE=true >> .env
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-          cache: 'pip'
-
-      - name: Setup Python Environment
-        run: |
-          make sdk-setup
-          source sdk/.venv/bin/activate
-
-      - name: SDK Release
-        run: make sdk-release
-
   ui-release:
     needs:
       - setup
@@ -91,7 +64,6 @@ jobs:
     needs:
       - setup
       - api-release
-      - sdk-release
       - ui-release
     runs-on: self-hosted
     steps:

.github/workflows/release_sdk.yml

@@ -0,0 +1,57 @@
+name: rAPId Release
+
+on:
+  release:
+    types: [released]
+
+jobs:
+  setup:
+    if: github.event.release.name == 'sdk release'
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Log commit SHA
+        run: echo $GITHUB_SHA
+
+  sdk-release:
+    needs:
+      - setup
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Populate .env with additional vars
+        run: |
+          echo TWINE_USERNAME=${{ secrets.TWINE_USERNAME }} >> .env
+          echo TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} >> .env
+          echo TWINE_NON_INTERACTIVE=true >> .env
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Setup Python Environment
+        run: |
+          make sdk-setup
+          source sdk/.venv/bin/activate
+
+      - name: SDK Release
+        run: make sdk-release
+
+  cleanup:
+    needs:
+      - setup
+      - sdk-release
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Clean Docker Context
+        if: always()
+        run: make clean-pipeline-docker-context

Makefile

@@ -186,20 +186,20 @@ ui-release:
 
 ui-zip-and-release: ui-zip-contents ui-release ## Zip and release prod static ui site
 
-
-##
 release:
-	@python release.py --operation check
+	@python release.py --operation check --type ${type}
 	@git checkout ${commit}
 	@git tag -a "${version}" -m "Release tag for version ${version}"
 	@git checkout -
 	@git push origin ${version}
-	@python release.py --operation create-changelog
-	@gh release create ${version} -F latest_release_changelog.md
-	@rm -rf latest_release_changelog.md
-
+	@python release.py --operation create-changelog --type ${type}
+	@gh release create ${version} -F latest_release_changelog_${type}.md -t "${type} release"
+	@rm -rf latest_release_changelog_${type}.md
 
 # Migration --------------------
 ##
 migrate-v7:			## Run the migration
 	@cd api/; ./batect migrate-v7 -- --layer ${layer} --all-layers ${all-layers}
+
+serve-docs:
+	mkdocs serve

api/api/adapter/s3_adapter.py

@@ -103,12 +103,14 @@ def get_last_updated_time(self, file_path: str) -> Optional[str]:
         paginator = self.__s3_client.get_paginator("list_objects_v2")
         page_iterator = paginator.paginate(Bucket=self.__s3_bucket, Prefix=file_path)
         try:
-            return max(
-                [
-                    item["LastModified"]
-                    for page in page_iterator
-                    for item in page["Contents"]
-                ]
+            return str(
+                max(
+                    [
+                        item["LastModified"]
+                        for page in page_iterator
+                        for item in page["Contents"]
+                    ]
+                )
             )
         except KeyError:
             return None

api/api/application/services/data_service.py

@@ -184,7 +184,7 @@ def remove_existing_data(self, schema: Schema, raw_file_identifier: str) -> None
 
     def get_last_updated_time(self, metadata: DatasetMetadata) -> str:
         last_updated = self.s3_adapter.get_last_updated_time(
-            metadata.s3_file_location()
+            metadata.dataset_location()
         )
         return last_updated or "Never updated"
 

api/api/common/config/constants.py

@@ -1,3 +1,5 @@
+import os
+
 BASE_API_PATH = "/api"
 BASE_REGEX = "^[a-zA-Z0-9_-]"
 FILENAME_WITH_TIMESTAMP_REGEX = r"[a-zA-Z0-9:_\-]+.csv$"
@@ -22,7 +24,6 @@
 )
 
 USERNAME_REGEX = "[a-zA-Z][a-zA-Z0-9@._-]{2,127}"
-
 DEFAULT_JOB_EXPIRY_DAYS = 1
 UPLOAD_JOB_EXPIRY_DAYS = 7
 QUERY_JOB_EXPIRY_DAYS = 1
@@ -39,3 +40,8 @@
 
 FIRST_SCHEMA_VERSION_NUMBER = 1
 SCHEMA_VERSION_INCREMENT = 1
+
+CUSTOM_USER_NAME_REGEX = os.getenv("CUSTOM_USER_NAME_REGEX")
+CUSTOM_USER_NAME_REGEX = (
+    None if CUSTOM_USER_NAME_REGEX == "" else CUSTOM_USER_NAME_REGEX
+)

api/api/domain/user.py

@@ -1,10 +1,13 @@
 import re
 from typing import Optional, List
-
 from pydantic import BaseModel
 
 from api.common.config.auth import DEFAULT_PERMISSION, ALLOWED_EMAIL_DOMAINS
-from api.common.config.constants import EMAIL_REGEX, USERNAME_REGEX
+from api.common.config.constants import (
+    EMAIL_REGEX,
+    USERNAME_REGEX,
+    CUSTOM_USER_NAME_REGEX,
+)
 from api.common.custom_exceptions import UserError
 
 
@@ -19,8 +22,18 @@ def get_validated_username(self):
         https://docs.aws.amazon.com/cognito/latest/developerguide/limits.html
         """
         if self.username is not None and re.fullmatch(USERNAME_REGEX, self.username):
-            return self.username
-        raise UserError("Invalid username provided")
+            if CUSTOM_USER_NAME_REGEX is None:
+                return self.username
+            else:
+                if re.fullmatch(CUSTOM_USER_NAME_REGEX, self.username):
+                    return self.username
+                raise UserError(
+                    "Your username does not match the requirements specified by your organisation."
+                    + CUSTOM_USER_NAME_REGEX
+                )
+        raise UserError(
+            "This username is invalid. Please check the username and try again"
+        )
 
     def get_permissions(self) -> List[str]:
         return self.permissions

api/batect.yml

@@ -23,7 +23,7 @@ containers:
       ALLOWED_EMAIL_DOMAINS: ${ALLOWED_EMAIL_DOMAINS:-}
       RESOURCE_PREFIX: ${RESOURCE_PREFIX:-prefix}
       LAYERS: ${LAYERS:-}
-
+      CUSTOM_USER_NAME_REGEX: ${CUSTOM_USER_NAME_REGEX:-}
 tasks:
   runtime-environment:
     description: Build runtime environment

api/requirements.txt

@@ -13,4 +13,4 @@ pydantic[email]
 python-multipart
 uvicorn
 requests
-strenum
+strenum

api/test/api/adapter/test_cognito_adapter.py

@@ -1,6 +1,6 @@
 from abc import ABC
 from unittest.mock import Mock
-
+from unittest import mock
 import pytest
 from botocore.exceptions import ClientError
 
@@ -248,6 +248,50 @@ def test_throws_error_when_client_app_name_has_not_been_changed_from_placeholder
 
 class TestCognitoAdapterUsers(BaseCognitoAdapter):
     def test_create_user(self):
+        cognito_response = {
+            "User": {
+                "Username": "TtestUser1",
+                "Attributes": [
+                    {"Name": "sub", "Value": "some-uu-id-b226-e5fd18c59b85"},
+                    {"Name": "email_verified", "Value": "True"},
+                    {"Name": "email", "Value": "user-name@example1.com"},
+                ],
+            },
+            "ResponseMetadata": {
+                "RequestId": "the-request-id-b368-fae5cebb746f",
+                "HTTPStatusCode": 200,
+            },
+        }
+        expected_response = UserResponse(
+            username="TtestUser1",
+            email="user-name@example1.com",
+            permissions=["WRITE_PUBLIC", "READ_PRIVATE"],
+            user_id="some-uu-id-b226-e5fd18c59b85",
+        )
+        request = UserRequest(
+            username="TtestUser1",
+            email="TtestUser1@example1.com",
+            permissions=["WRITE_PUBLIC", "READ_PRIVATE"],
+        )
+        self.cognito_boto_client.admin_create_user.return_value = cognito_response
+
+        actual_response = self.cognito_adapter.create_user(request)
+        self.cognito_boto_client.admin_create_user.assert_called_once_with(
+            UserPoolId=COGNITO_USER_POOL_ID,
+            Username="TtestUser1",
+            UserAttributes=[
+                {"Name": "email", "Value": "TtestUser1@example1.com"},
+                {"Name": "email_verified", "Value": "True"},
+            ],
+            DesiredDeliveryMediums=[
+                "EMAIL",
+            ],
+        )
+
+        assert actual_response == expected_response
+
+    @mock.patch("api.domain.user.CUSTOM_USER_NAME_REGEX", None)
+    def test_create_user_no_custom_regex(self):
         cognito_response = {
             "User": {
                 "Username": "user-name",
@@ -322,8 +366,8 @@ def test_delete_user_fails_when_user_does_not_exist(self):
 
     def test_create_user_fails_in_aws(self):
         request = UserRequest(
-            username="user-name",
-            email="user-name@example1.com",
+            username="TtestUser1",
+            email="TtestUser1@example1.com",
             permissions=["WRITE_PUBLIC", "READ_PRIVATE"],
         )
 
@@ -333,14 +377,14 @@ def test_create_user_fails_in_aws(self):
         )
 
         with pytest.raises(
-            AWSServiceError, match="The user 'user-name' could not be created"
+            AWSServiceError, match="The user 'TtestUser1' could not be created"
         ):
             self.cognito_adapter.create_user(request)
 
     def test_create_user_fails_when_the_user_already_exist(self):
         request = UserRequest(
-            username="user-name",
-            email="user-name@example1.com",
+            username="TtestUser1",
+            email="TtestUser1@example1.com",
             permissions=["WRITE_PUBLIC", "READ_PRIVATE"],
         )
 
@@ -351,7 +395,7 @@ def test_create_user_fails_when_the_user_already_exist(self):
 
         with pytest.raises(
             UserError,
-            match="The user 'user-name' or email 'user-name@example1.com' already exist",
+            match="The user 'TtestUser1' or email 'TtestUser1@example1.com' already exist",
         ):
             self.cognito_adapter.create_user(request)
 

api/test/api/application/services/test_data_service.py

@@ -613,7 +613,7 @@ def test_get_last_updated_time(self):
         )
         assert last_updated_time == "2022-03-01 11:03:49+00:00"
         self.s3_adapter.get_last_updated_time.assert_called_once_with(
-            self.valid_schema.metadata.s3_file_location()
+            self.valid_schema.metadata.dataset_location()
         )
 
     def test_get_last_updated_time_empty(self):
@@ -624,7 +624,7 @@ def test_get_last_updated_time_empty(self):
         )
         assert last_updated_time == "Never updated"
         self.s3_adapter.get_last_updated_time.assert_called_once_with(
-            self.valid_schema.metadata.s3_file_location()
+            self.valid_schema.metadata.dataset_location()
         )
 
     def test_get_schema_information(self):