diff --git a/.env.example b/.env.example index 95657e4..ba1557f 100644 --- a/.env.example +++ b/.env.example @@ -8,12 +8,11 @@ RESOURCE_PREFIX=rapid DOMAIN_NAME=example.com COGNITO_USER_POOL_ID=11111111 LAYERS=raw,layer - +CUSTOM_USER_NAME_REGEX="regex_expression" # SDK Specific RAPID_CLIENT_ID= RAPID_CLIENT_SECRET= RAPID_URL= - # UI Specific NEXT_PUBLIC_API_URL= NEXT_PUBLIC_API_URL_PROXY= diff --git a/.github/.github.env b/.github/.github.env index ad8c623..5702873 100644 --- a/.github/.github.env +++ b/.github/.github.env @@ -4,3 +4,4 @@ ALLOWED_EMAIL_DOMAINS=example1.com,example2.com LAYERS=raw,layer DOMAIN_NAME=example.com DATA_BUCKET=the-bucket +CUSTOM_USER_NAME_REGEX=[a-zA-Z][a-zA-Z0-9@._-]{2,127} diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index a12d546..a524262 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -90,10 +90,12 @@ jobs: - name: SDK Test run: make sdk-test - # TODO: Add back in - # - name: SDK Test Deploy - # run: make sdk-release-test + - name: Set env variable + run: echo "TEST_SDK_VERSION=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV + - name: SDK Test Deploy + run: make sdk-release-test + ui-dev: needs: - setup diff --git a/.github/workflows/release.yml b/.github/workflows/release_api.yml similarity index 68% rename from .github/workflows/release.yml rename to .github/workflows/release_api.yml index f1220b2..486c4e3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release_api.yml @@ -6,6 +6,7 @@ on: jobs: setup: + if: github.event.release.name == 'api release' runs-on: self-hosted steps: - name: Checkout @@ -34,34 +35,6 @@ jobs: - name: API Tag and Upload Release Image run: make api-tag-and-upload-release-image - sdk-release: - needs: - - setup - runs-on: self-hosted - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Populate .env with additional vars - run: | - echo TWINE_USERNAME=${{ secrets.TWINE_USERNAME }} >> .env - echo TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} >> .env - echo TWINE_NON_INTERACTIVE=true >> .env - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - cache: 'pip' - - - name: Setup Python Environment - run: | - make sdk-setup - source sdk/.venv/bin/activate - - - name: SDK Release - run: make sdk-release - ui-release: needs: - setup @@ -91,7 +64,6 @@ jobs: needs: - setup - api-release - - sdk-release - ui-release runs-on: self-hosted steps: diff --git a/.github/workflows/release_sdk.yml b/.github/workflows/release_sdk.yml new file mode 100644 index 0000000..df17650 --- /dev/null +++ b/.github/workflows/release_sdk.yml @@ -0,0 +1,57 @@ +name: rAPId Release + +on: + release: + types: [released] + +jobs: + setup: + if: github.event.release.name == 'sdk release' + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Log commit SHA + run: echo $GITHUB_SHA + + sdk-release: + needs: + - setup + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Populate .env with additional vars + run: | + echo TWINE_USERNAME=${{ secrets.TWINE_USERNAME }} >> .env + echo TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} >> .env + echo TWINE_NON_INTERACTIVE=true >> .env + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + + - name: Setup Python Environment + run: | + make sdk-setup + source sdk/.venv/bin/activate + + - name: SDK Release + run: make sdk-release + + cleanup: + needs: + - setup + - sdk-release + runs-on: self-hosted + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Clean Docker Context + if: always() + run: make clean-pipeline-docker-context diff --git a/Makefile b/Makefile index 72c138f..55d71ca 100644 --- a/Makefile +++ b/Makefile @@ -186,20 +186,20 @@ ui-release: ui-zip-and-release: ui-zip-contents ui-release ## Zip and release prod static ui site - -## release: - @python release.py --operation check + @python release.py --operation check --type ${type} @git checkout ${commit} @git tag -a "${version}" -m "Release tag for version ${version}" @git checkout - @git push origin ${version} - @python release.py --operation create-changelog - @gh release create ${version} -F latest_release_changelog.md - @rm -rf latest_release_changelog.md - + @python release.py --operation create-changelog --type ${type} + @gh release create ${version} -F latest_release_changelog_${type}.md -t "${type} release" + @rm -rf latest_release_changelog_${type}.md # Migration -------------------- ## migrate-v7: ## Run the migration @cd api/; ./batect migrate-v7 -- --layer ${layer} --all-layers ${all-layers} + +serve-docs: + mkdocs serve \ No newline at end of file diff --git a/api/api/adapter/s3_adapter.py b/api/api/adapter/s3_adapter.py index 4bcd7d7..a69beea 100644 --- a/api/api/adapter/s3_adapter.py +++ b/api/api/adapter/s3_adapter.py @@ -103,12 +103,14 @@ def get_last_updated_time(self, file_path: str) -> Optional[str]: paginator = self.__s3_client.get_paginator("list_objects_v2") page_iterator = paginator.paginate(Bucket=self.__s3_bucket, Prefix=file_path) try: - return max( - [ - item["LastModified"] - for page in page_iterator - for item in page["Contents"] - ] + return str( + max( + [ + item["LastModified"] + for page in page_iterator + for item in page["Contents"] + ] + ) ) except KeyError: return None diff --git a/api/api/application/services/data_service.py b/api/api/application/services/data_service.py index 1f81c8c..37d8d7e 100644 --- a/api/api/application/services/data_service.py +++ b/api/api/application/services/data_service.py @@ -184,7 +184,7 @@ def remove_existing_data(self, schema: Schema, raw_file_identifier: str) -> None def get_last_updated_time(self, metadata: DatasetMetadata) -> str: last_updated = self.s3_adapter.get_last_updated_time( - metadata.s3_file_location() + metadata.dataset_location() ) return last_updated or "Never updated" diff --git a/api/api/common/config/constants.py b/api/api/common/config/constants.py index 60f6ccb..a50a9ce 100644 --- a/api/api/common/config/constants.py +++ b/api/api/common/config/constants.py @@ -1,3 +1,5 @@ +import os + BASE_API_PATH = "/api" BASE_REGEX = "^[a-zA-Z0-9_-]" FILENAME_WITH_TIMESTAMP_REGEX = r"[a-zA-Z0-9:_\-]+.csv$" @@ -22,7 +24,6 @@ ) USERNAME_REGEX = "[a-zA-Z][a-zA-Z0-9@._-]{2,127}" - DEFAULT_JOB_EXPIRY_DAYS = 1 UPLOAD_JOB_EXPIRY_DAYS = 7 QUERY_JOB_EXPIRY_DAYS = 1 @@ -39,3 +40,8 @@ FIRST_SCHEMA_VERSION_NUMBER = 1 SCHEMA_VERSION_INCREMENT = 1 + +CUSTOM_USER_NAME_REGEX = os.getenv("CUSTOM_USER_NAME_REGEX") +CUSTOM_USER_NAME_REGEX = ( + None if CUSTOM_USER_NAME_REGEX == "" else CUSTOM_USER_NAME_REGEX +) diff --git a/api/api/domain/user.py b/api/api/domain/user.py index e593a4d..a012240 100644 --- a/api/api/domain/user.py +++ b/api/api/domain/user.py @@ -1,10 +1,13 @@ import re from typing import Optional, List - from pydantic import BaseModel from api.common.config.auth import DEFAULT_PERMISSION, ALLOWED_EMAIL_DOMAINS -from api.common.config.constants import EMAIL_REGEX, USERNAME_REGEX +from api.common.config.constants import ( + EMAIL_REGEX, + USERNAME_REGEX, + CUSTOM_USER_NAME_REGEX, +) from api.common.custom_exceptions import UserError @@ -19,8 +22,18 @@ def get_validated_username(self): https://docs.aws.amazon.com/cognito/latest/developerguide/limits.html """ if self.username is not None and re.fullmatch(USERNAME_REGEX, self.username): - return self.username - raise UserError("Invalid username provided") + if CUSTOM_USER_NAME_REGEX is None: + return self.username + else: + if re.fullmatch(CUSTOM_USER_NAME_REGEX, self.username): + return self.username + raise UserError( + "Your username does not match the requirements specified by your organisation." + + CUSTOM_USER_NAME_REGEX + ) + raise UserError( + "This username is invalid. Please check the username and try again" + ) def get_permissions(self) -> List[str]: return self.permissions diff --git a/api/batect.yml b/api/batect.yml index de84076..5daa08f 100755 --- a/api/batect.yml +++ b/api/batect.yml @@ -23,7 +23,7 @@ containers: ALLOWED_EMAIL_DOMAINS: ${ALLOWED_EMAIL_DOMAINS:-} RESOURCE_PREFIX: ${RESOURCE_PREFIX:-prefix} LAYERS: ${LAYERS:-} - + CUSTOM_USER_NAME_REGEX: ${CUSTOM_USER_NAME_REGEX:-} tasks: runtime-environment: description: Build runtime environment diff --git a/api/requirements.txt b/api/requirements.txt index 440de54..b22e35f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -13,4 +13,4 @@ pydantic[email] python-multipart uvicorn requests -strenum +strenum \ No newline at end of file diff --git a/api/test/api/adapter/test_cognito_adapter.py b/api/test/api/adapter/test_cognito_adapter.py index a43a7b8..4e94d25 100644 --- a/api/test/api/adapter/test_cognito_adapter.py +++ b/api/test/api/adapter/test_cognito_adapter.py @@ -1,6 +1,6 @@ from abc import ABC from unittest.mock import Mock - +from unittest import mock import pytest from botocore.exceptions import ClientError @@ -248,6 +248,50 @@ def test_throws_error_when_client_app_name_has_not_been_changed_from_placeholder class TestCognitoAdapterUsers(BaseCognitoAdapter): def test_create_user(self): + cognito_response = { + "User": { + "Username": "TtestUser1", + "Attributes": [ + {"Name": "sub", "Value": "some-uu-id-b226-e5fd18c59b85"}, + {"Name": "email_verified", "Value": "True"}, + {"Name": "email", "Value": "user-name@example1.com"}, + ], + }, + "ResponseMetadata": { + "RequestId": "the-request-id-b368-fae5cebb746f", + "HTTPStatusCode": 200, + }, + } + expected_response = UserResponse( + username="TtestUser1", + email="user-name@example1.com", + permissions=["WRITE_PUBLIC", "READ_PRIVATE"], + user_id="some-uu-id-b226-e5fd18c59b85", + ) + request = UserRequest( + username="TtestUser1", + email="TtestUser1@example1.com", + permissions=["WRITE_PUBLIC", "READ_PRIVATE"], + ) + self.cognito_boto_client.admin_create_user.return_value = cognito_response + + actual_response = self.cognito_adapter.create_user(request) + self.cognito_boto_client.admin_create_user.assert_called_once_with( + UserPoolId=COGNITO_USER_POOL_ID, + Username="TtestUser1", + UserAttributes=[ + {"Name": "email", "Value": "TtestUser1@example1.com"}, + {"Name": "email_verified", "Value": "True"}, + ], + DesiredDeliveryMediums=[ + "EMAIL", + ], + ) + + assert actual_response == expected_response + + @mock.patch("api.domain.user.CUSTOM_USER_NAME_REGEX", None) + def test_create_user_no_custom_regex(self): cognito_response = { "User": { "Username": "user-name", @@ -322,8 +366,8 @@ def test_delete_user_fails_when_user_does_not_exist(self): def test_create_user_fails_in_aws(self): request = UserRequest( - username="user-name", - email="user-name@example1.com", + username="TtestUser1", + email="TtestUser1@example1.com", permissions=["WRITE_PUBLIC", "READ_PRIVATE"], ) @@ -333,14 +377,14 @@ def test_create_user_fails_in_aws(self): ) with pytest.raises( - AWSServiceError, match="The user 'user-name' could not be created" + AWSServiceError, match="The user 'TtestUser1' could not be created" ): self.cognito_adapter.create_user(request) def test_create_user_fails_when_the_user_already_exist(self): request = UserRequest( - username="user-name", - email="user-name@example1.com", + username="TtestUser1", + email="TtestUser1@example1.com", permissions=["WRITE_PUBLIC", "READ_PRIVATE"], ) @@ -351,7 +395,7 @@ def test_create_user_fails_when_the_user_already_exist(self): with pytest.raises( UserError, - match="The user 'user-name' or email 'user-name@example1.com' already exist", + match="The user 'TtestUser1' or email 'TtestUser1@example1.com' already exist", ): self.cognito_adapter.create_user(request) diff --git a/api/test/api/application/services/test_data_service.py b/api/test/api/application/services/test_data_service.py index c2af490..d5b9275 100644 --- a/api/test/api/application/services/test_data_service.py +++ b/api/test/api/application/services/test_data_service.py @@ -613,7 +613,7 @@ def test_get_last_updated_time(self): ) assert last_updated_time == "2022-03-01 11:03:49+00:00" self.s3_adapter.get_last_updated_time.assert_called_once_with( - self.valid_schema.metadata.s3_file_location() + self.valid_schema.metadata.dataset_location() ) def test_get_last_updated_time_empty(self): @@ -624,7 +624,7 @@ def test_get_last_updated_time_empty(self): ) assert last_updated_time == "Never updated" self.s3_adapter.get_last_updated_time.assert_called_once_with( - self.valid_schema.metadata.s3_file_location() + self.valid_schema.metadata.dataset_location() ) def test_get_schema_information(self): diff --git a/api/test/api/controller/test_schema.py b/api/test/api/controller/test_schema.py index f979dc9..b7c586f 100644 --- a/api/test/api/controller/test_schema.py +++ b/api/test/api/controller/test_schema.py @@ -61,10 +61,10 @@ def test_return_400_pydantic_error(self): assert response.status_code == 400 assert response.json() == { "details": [ - "layer -> Field required", - "domain -> Field required", - "dataset -> Field required", - "sensitivity -> Field required", + "metadata: layer -> Field required", + "metadata: domain -> Field required", + "metadata: dataset -> Field required", + "metadata: sensitivity -> Field required", ] } @@ -232,10 +232,10 @@ def test_return_400_when_request_body_invalid(self): assert response.status_code == 400 assert response.json() == { "details": [ - "layer -> Field required", - "domain -> Field required", - "dataset -> Field required", - "sensitivity -> Field required", + "metadata: layer -> Field required", + "metadata: domain -> Field required", + "metadata: dataset -> Field required", + "metadata: sensitivity -> Field required", ] } diff --git a/api/test/api/domain/test_user_request.py b/api/test/api/domain/test_user_request.py index 035f13c..d5bdbb3 100644 --- a/api/test/api/domain/test_user_request.py +++ b/api/test/api/domain/test_user_request.py @@ -2,6 +2,7 @@ from api.common.custom_exceptions import UserError from api.domain.user import UserRequest +from unittest import mock class TestUserRequest: @@ -16,9 +17,13 @@ class TestUserRequest: "S1234", ], ) + @mock.patch( + "api.domain.user.CUSTOM_USER_NAME_REGEX", "[a-zA-Z][a-zA-Z0-9@._-]{2,127}" + ) def test_get_validated_username(self, provided_username): request = UserRequest(username=provided_username, email="user@email.com") + # Overwrite env variable on fn import try: validated_name = request.get_validated_username() assert validated_name == provided_username @@ -40,10 +45,55 @@ def test_get_validated_username(self, provided_username): "A" * 129, ], ) + @mock.patch( + "api.domain.user.CUSTOM_USER_NAME_REGEX", "[a-zA-Z][a-zA-Z0-9@._-]{2,127}" + ) def test_raises_error_when_invalid_username(self, provided_username): request = UserRequest(username=provided_username, email="user@email.com") + # Overrwrite env variable on fn import + with pytest.raises( + UserError, + match="This username is invalid. Please check the username and try again", + ): + request.get_validated_username() + + @pytest.mark.parametrize( + "provided_username", + [ + "Ttest123", + "TCheck", + "SamCheck", + "Sam123Check456", + "S1234", + ], + ) + @mock.patch("api.domain.user.CUSTOM_USER_NAME_REGEX", "^[A-Z][A-Za-z0-9]{3,50}$") + def test_get_validated_username_custom_regex(self, provided_username): + request = UserRequest(username=provided_username, email="user@email.com") + try: + validated_name = request.get_validated_username() + assert validated_name == provided_username + except UserError: + pytest.fail("An unexpected UserError was thrown") + + @pytest.mark.parametrize( + "provided_username", + [ + "username_name", + "username_name2", + "username@email.com", + "VA.li_d@na-mE", + "A....", + ], + ) + @mock.patch("api.domain.user.CUSTOM_USER_NAME_REGEX", "^[A-Z][A-Za-z0-9]{3,50}$") + def test_raises_error_when_invalid_username_custom_regex(self, provided_username): + request = UserRequest(username=provided_username, email="user@email.com") - with pytest.raises(UserError, match="Invalid username provided"): + with pytest.raises( + UserError, + match="Your username does not match the requirements specified by your organisation", + ): request.get_validated_username() @pytest.mark.parametrize( diff --git a/docs/changelog.md b/docs/changelog/api.md similarity index 81% rename from docs/changelog.md rename to docs/changelog/api.md index a0cc5ad..5d12c92 100644 --- a/docs/changelog.md +++ b/docs/changelog/api.md @@ -1,12 +1,13 @@ +# API Changelog + # Changelog -## v7.0.8 / v0.1.6 (sdk) - _2023-11-15_ +## v7.0.8 - _2023-11-15_ ### Fixes - Issue with date types when editing a schema on the UI because of no option to apply format column and therefore getting an _all fields are required_ error. - Tweaked UI design when adding permissions to subject. -- SDK not uploading a Pandas Dataframe with a date field set correctly. - Updated NextJS and Zod package version. ### Features @@ -17,11 +18,10 @@ - https://github.com/no10ds/rapid/issues/57 -## v7.0.7 / v0.1.5 (sdk) - _2023-11-07_ +## v7.0.7 - _2023-11-07_ ### Fixes -- Issue within the sdk `upload_and_create_dataset` function where schema metadata wasn't being correctly overridden. - Hitting maximum security group rules for the load balancer. - Documentation improvements and removes any references to the old deprecated repositories. @@ -32,7 +32,7 @@ - https://github.com/no10ds/rapid/issues/54 - https://github.com/no10ds/rapid/issues/51 -## v7.0.6 / v0.1.4 (sdk) - _2023-10-18_ +## v7.0.6 - _2023-10-18_ ### Features @@ -41,17 +41,10 @@ ### Fixes -- Fixed an issue with the sdk not showing schemas were created successfully due to a wrong response code. - Where dataset info was being called on columns with a date type, this was causing an issue with the Pydantic validation. - Tweaked the documentation to implement searching for column heading style guide to match what the API returns in the error message. -## v7.0.5 / v0.1.3 (sdk) - _2023-09-20_ - -### Fixes - -- Fix the behaviour of the dataset pattern functions in the SDK. - -## v7.0.4 / v0.1.2 (sdk) - _2023-09-20_ +## v7.0.4 - _2023-09-20_ ### Features @@ -63,26 +56,25 @@ - Updated terraform default `application_version` and `ui_version` variables. - Migration script and documentation. -## v7.0.3 / v0.1.2 (sdk) - _2023-09-15_ +## v7.0.3 - _2023-09-15_ ### Fixes - Fixes issue where permissions were not being correctly read and causing api functionality to fail -## v7.0.2 / v0.1.2 (sdk) - _2023-09-14_ +## v7.0.2 - _2023-09-14_ ### Fixes - Update UI repo references. -## v7.0.1 / v0.1.2 (sdk) - _2023-09-13_ +## v7.0.1 - _2023-09-13_ ### Fixes - Date types were being stored as strings which caused issues when querying with Athena. They are now stored as date types. -- Rename the rAPId sdk method `generate_info` to `fetch_dataset_info` and remove an unnecessary argument. -## v7.0.0 / v0.1.1 (sdk) - _2023-09-12_ +## v7.0.0 - _2023-09-12_ ### Features @@ -96,7 +88,6 @@ ### Breaking Changes - All dataset endpoints will be prefixed with `layer`. Typically going from `domain/dataset` to `layer/domain/dataset`. -- All sdk functions that interact with datasets will now require an argument for layer. ### Migration diff --git a/docs/changelog/sdk.md b/docs/changelog/sdk.md new file mode 100644 index 0000000..be0fc57 --- /dev/null +++ b/docs/changelog/sdk.md @@ -0,0 +1,75 @@ +# SDK Changelog + +## v0.1.6 - _2023-11-15_ + +### Fixes + +- SDK not uploading a Pandas Dataframe with a date field set correctly. + +### Closes relevant GitHub issues + +- https://github.com/no10ds/rapid/issues/57 + +## v0.1.5 - _2023-11-07_ + +### Fixes + +- Issue within the sdk `upload_and_create_dataset` function where schema metadata wasn't being correctly overridden. +- Documentation improvements and removes any references to the old deprecated repositories. + + +## v0.1.4 - _2023-10-18_ + +### Features + +- Clients can now be created and deleted via the sdk. + +### Fixes + +- Fixed an issue with the sdk not showing schemas were created successfully due to a wrong response code. +- Where dataset info was being called on columns with a date type, this was causing an issue with the Pydantic validation. +- Tweaked the documentation to implement searching for column heading style guide to match what the API returns in the error message. + +## v0.1.3 - _2023-09-20_ + +### Fixes + +- Fix the behaviour of the dataset pattern functions in the SDK. + +## v0.1.2 - _2023-09-13_ + +### Fixes + +- Date types were being stored as strings which caused issues when querying with Athena. They are now stored as date types. +- Rename the rAPId sdk method `generate_info` to `fetch_dataset_info` and remove an unnecessary argument. + +## v0.1.1- _2023-09-12_ + +### Features + +- Layers have been introduced to rAPId. These are now the highest level of grouping for your data. They allow you to separate your data into areas that relate to the layers in your data architecture e.g `raw`, `curated`, `presentation`. You will need to specify your layers when you create or migrate a rAPId instance. +- All the code is now in this monorepo. The previous [Infrastructure](https://github.com/no10ds/rapid-infrastructure), [UI](https://github.com/no10ds/rapid-ui) and [API](https://github.com/no10ds/rapid-api) repos are now deprecated. This will ease the use and development of rAPId. +- Schemas are now stored in DynamoDB, rather than S3. This offers speed and usability improvements, as well as making rAPId easier to extend. +- Code efficiency improvements. There were several areas in rAPId where we were executing costly operations that caused performance to degrade at scale. We've fixed these inefficiencies, taking us from O(n²) -> O(n) in these areas. +- Glue Crawlers have been removed, with Athena tables are created directly by the API instead. Data is now available to query immediately after it is uploaded, rather than the previous wait (approximately 3 mins) while crawlers ran. It also offers scalability benefits because without crawlers we are not dependant on the number of free IPs within the subnet. +- Improved UI testing with Playwright. + +### Breaking Changes + +- All dataset endpoints will be prefixed with `layer`. Typically going from `domain/dataset` to `layer/domain/dataset`. +- All sdk functions that interact with datasets will now require an argument for layer. + +### Migration + +- See the [migration doc](migration.md) for details on how to migrate to v7 from v6. + +[Unreleased changes]: https://github.com/no10ds/rapid/compare/v7.0.8...HEAD +[v7.0.8 / v0.1.6 (sdk)]: https://github.com/no10ds/rapid/v7.0.7...v7.0.8 +[v7.0.7 / v0.1.5 (sdk)]: https://github.com/no10ds/rapid/v7.0.6...v7.0.7 +[v7.0.6 / v0.1.4 (sdk)]: https://github.com/no10ds/rapid/v7.0.5...v7.0.6 +[v7.0.5 / v0.1.3 (sdk)]: https://github.com/no10ds/rapid/v7.0.4...v7.0.5 +[v7.0.4 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.3...v7.0.4 +[v7.0.3 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.2...v7.0.3 +[v7.0.2 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.1...v7.0.2 +[v7.0.1 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.0...v7.0.1 +[v7.0.0 / v0.1.1 (sdk)]: https://github.com/no10ds/rapid/v7.0.0 diff --git a/docs/contributing.md b/docs/contributing.md index 886444b..7f5c1ad 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -163,7 +163,7 @@ rAPId has several core components that are all versioned when a release is creat 3. A public pypi package of the built rAPId-sdk that provides an easy and pythonic way to interact with the API. 4. A terraform module that can be used to create the AWS services necessary for running the various components of rAPId. -Performing a release involves tagging the repository with a new version number so that the API image, UI, SDK and Terraform all get versioned. +Performing a release involves tagging the repository with a new version number so that the API image, UI, SDK and Terraform all get versioned. Note that the API and SDK can be versioned and released independently, so you can conserve the API version number if just releasing an update to the SDK. ### Prerequisites @@ -172,11 +172,13 @@ Performing a release involves tagging the repository with a new version number s ### Steps -1. Decide on the new version number following the [semantic versioning approach](https://semver.org/) +1. Decide on the new version number for the API and the UI and/or the SDK following the [semantic versioning approach](https://semver.org/). 2. Update and commit the Changelog (you can follow - the [template](https://github.com/no10ds/rapid/blob/main/changelog_release_template/md)) -3. Run `make release commit= version=vX.X.X` + the [template](https://github.com/no10ds/rapid/blob/main/changelog_release_template/md)). You'll need to separate SDK and API changes into their respective changelogs, under docs/changelog. + 1. Bundle API, UI and terraform changes as part of the API changelog. + 2. Insert SDK changes into the SDK changelog. +3. Run `make release commit= type= version=vX.X.X` -> Ensure the version number follows the format `vX.X.X` with full-stops in the same places +> Ensure the version number follows the format `vX.X.X` with full-stops in the same places for both API and SDK changes. Now the release pipeline will run automatically, build all images and packages off that version of the code and tag it within GitHub. diff --git a/docs/infrastructure/deployment.md b/docs/infrastructure/deployment.md index 01143ea..a922b5b 100644 --- a/docs/infrastructure/deployment.md +++ b/docs/infrastructure/deployment.md @@ -56,6 +56,9 @@ There are also these optional inputs: - `catalog_disabled` - if set to `true` it will disable the rAPId internal data catalogue - `tags` - if provided, it will tag the resources with the defined value. Otherwise, it will default to "Resource = ' data-f1-rapid'" +- `custom_user_name_regex` - Regex that when supplied usernames must conform to when creating a new user. Defaults to none, in which case rAPId will default to it's basic username validity checks. +- `task_cpu` - If provided, will update CPU resource allocated to the ECS task running rAPId instance. Otherwise will default to 256. +- `task_memory` - If provided, will update memory resource allocated to the ECS task running rAPId instance. Otherwise will default to 512. Once you apply the Terraform, a new instance of the application should be created. @@ -104,6 +107,8 @@ certificate_validation_arn = "" support_emails_for_cloudwatch_alerts = [] tags = {} +task_memory = "" +task_cpu = "" ``` `backend.hcl` template: diff --git a/infrastructure/blocks/app-cluster/main.tf b/infrastructure/blocks/app-cluster/main.tf index 874ab7b..f6125b8 100644 --- a/infrastructure/blocks/app-cluster/main.tf +++ b/infrastructure/blocks/app-cluster/main.tf @@ -39,6 +39,8 @@ module "app_cluster" { enable_cloudtrail = var.enable_cloudtrail support_emails_for_cloudwatch_alerts = var.support_emails_for_cloudwatch_alerts tags = var.tags + task_cpu = var.task_cpu + task_memory = var.task_memory } data "terraform_remote_state" "vpc-state" { diff --git a/infrastructure/blocks/app-cluster/variables.tf b/infrastructure/blocks/app-cluster/variables.tf index f9683a7..55a5eba 100644 --- a/infrastructure/blocks/app-cluster/variables.tf +++ b/infrastructure/blocks/app-cluster/variables.tf @@ -104,3 +104,15 @@ variable "layers" { description = "A list of the layers that the rAPId instance will contain" default = ["default"] } + +variable "task_memory" { + type = number + description = "rAPId ecs task memory" + default = 512 +} + +variable "task_cpu" { + type = number + description = "rAPId ecs task cpu" + default = 256 +} diff --git a/infrastructure/modules/app-cluster/main.tf b/infrastructure/modules/app-cluster/main.tf index 2d8e424..26cf208 100644 --- a/infrastructure/modules/app-cluster/main.tf +++ b/infrastructure/modules/app-cluster/main.tf @@ -9,6 +9,7 @@ locals { "COGNITO_USER_POOL_ID" : var.cognito_user_pool_id, "RESOURCE_PREFIX" : var.resource-name-prefix, "COGNITO_USER_LOGIN_APP_CREDENTIALS_SECRETS_NAME" : var.cognito_user_login_app_credentials_secrets_name + "CUSTOM_USER_NAME_REGEX" : var.custom_user_name_regex }, var.project_information) } @@ -321,8 +322,8 @@ resource "aws_ecs_task_definition" "aws-ecs-task" { "protocol": "${var.protocol}" } ], - "cpu": 256, - "memory": 512, + "cpu": ${var.task_cpu}, + "memory": ${var.task_memory}, "networkMode": "awsvpc" } ] @@ -330,8 +331,8 @@ resource "aws_ecs_task_definition" "aws-ecs-task" { requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" -memory = "512" -cpu = "256" +memory = var.task_memory +cpu = var.task_cpu execution_role_arn = aws_iam_role.ecsTaskExecutionRole.arn task_role_arn = aws_iam_role.ecsTaskExecutionRole.arn diff --git a/infrastructure/modules/app-cluster/variables.tf b/infrastructure/modules/app-cluster/variables.tf index 47e7b7c..9232c48 100644 --- a/infrastructure/modules/app-cluster/variables.tf +++ b/infrastructure/modules/app-cluster/variables.tf @@ -202,3 +202,22 @@ variable "layers" { description = "A list of the layers that the rAPId instance will contain" default = ["default"] } + +variable "custom_user_name_regex" { + type = string + description = "A regex expression for conditional user validation." + default = null + nullable = true +} + +variable "task_memory" { + type = number + description = "rAPId ecs task memory" + default = 512 +} + +variable "task_cpu" { + type = number + description = "rAPId ecs task cpu" + default = 256 +} diff --git a/infrastructure/modules/rapid/main.tf b/infrastructure/modules/rapid/main.tf index 1a57b1e..383b97b 100644 --- a/infrastructure/modules/rapid/main.tf +++ b/infrastructure/modules/rapid/main.tf @@ -31,6 +31,9 @@ module "app_cluster" { ecs_cluster_arn = var.ecs_cluster_arn ecs_cluster_name = var.ecs_cluster_name ecs_cluster_id = var.ecs_cluster_id + custom_user_name_regex = var.custom_user_name_regex + task_cpu = var.task_cpu + task_memory = var.task_memory } module "auth" { diff --git a/infrastructure/modules/rapid/variables.tf b/infrastructure/modules/rapid/variables.tf index ea04014..ff4b065 100644 --- a/infrastructure/modules/rapid/variables.tf +++ b/infrastructure/modules/rapid/variables.tf @@ -165,3 +165,22 @@ variable "layers" { description = "A list of the layers that the rAPId instance will contain" default = ["default"] } + +variable "custom_user_name_regex" { + type = string + description = "A regex expression for conditional user validation." + default = null + nullable = true +} + +variable "task_memory" { + type = number + description = "rAPId ecs task memory" + default = 512 +} + +variable "task_cpu" { + type = number + description = "rAPId ecs task cpu" + default = 256 +} diff --git a/mkdocs.yml b/mkdocs.yml index 435222f..ec40b8e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,10 +47,11 @@ nav: - sdk/api/items/schema.md - Patterns: - sdk/api/patterns/data.md - - Releases: changelog.md - Migration: migration.md - Contributing: contributing.md - + - Changelog: + - changelog/sdk.md + - changelog/api.md plugins: - search - mkdocstrings: diff --git a/release.py b/release.py index c2a410d..b0a9b89 100644 --- a/release.py +++ b/release.py @@ -2,8 +2,8 @@ import argparse -def create_changelog(): - with open("./docs/changelog.md", "r") as changelog_file: +def create_changelog(type): + with open(f"./docs/changelog/{type}.md", "r") as changelog_file: changelog_lines = changelog_file.readlines() parsed_lines = [] @@ -25,7 +25,7 @@ def create_changelog(): "It looks like there is no release information in the changelog. Please check it." ) else: - with open("latest_release_changelog.md", "w+") as latest_changelog: + with open(f"latest_release_changelog_{type}.md", "w+") as latest_changelog: latest_changelog.writelines(parsed_lines) @@ -41,9 +41,13 @@ def ask_yes_no_question(question): print("Please answer yes or no.") -def check(): - ask_yes_no_question("Have you updated the application and ui version in Terraform?") - ask_yes_no_question("Have you updated the changelog for this release?") +def check(type): + if type == "api": + ask_yes_no_question("Have you updated the API, UI and Terraform version?") + ask_yes_no_question("Have you updated the changelog for this release?") + elif type == "sdk": + ask_yes_no_question("Have you updated the changelog for this release?") + ask_yes_no_question("Have you updated the SDK?") if __name__ == "__main__": @@ -55,9 +59,18 @@ def check(): required=True, choices=["check", "create-changelog"], ) + + parser.add_argument( + "--type", + help="What are you releasing?", + required=True, + choices=["api", "sdk"], + ) + args = parser.parse_args() + match args.operation: case "create-changelog": - create_changelog() + create_changelog(args.type) case "check": check() diff --git a/sdk/rapid/exceptions.py b/sdk/rapid/exceptions.py index d519227..ad10646 100644 --- a/sdk/rapid/exceptions.py +++ b/sdk/rapid/exceptions.py @@ -84,3 +84,11 @@ class SubjectNotFoundException(Exception): class SubjectDeletionFailedException(Exception): pass + + +class InvalidDomainNameException(Exception): + pass + + +class DomainConflictException(Exception): + pass diff --git a/sdk/rapid/rapid.py b/sdk/rapid/rapid.py index 77c8e06..c72ea13 100644 --- a/sdk/rapid/rapid.py +++ b/sdk/rapid/rapid.py @@ -27,6 +27,8 @@ InvalidPermissionsException, SubjectAlreadyExistsException, SubjectNotFoundException, + InvalidDomainNameException, + DomainConflictException, ) @@ -413,3 +415,29 @@ def update_subject_permissions(self, subject_id: str, permissions: list[str]): raise InvalidPermissionsException( "One or more of the provided permissions is invalid or duplicated" ) + + def create_protected_domain(self, name: str): + """ + Creates a new protected domain. + + Args: + name (str): The name of the protected domain to create. + + Raises: + rapid.exceptions.InvalidDomainNameException: If the domain name is invalid. + rapid.exceptions.DomainConflictException: If the domain already exists. + + """ + url = f"{self.auth.url}/protected_domains/{name}" + response = requests.post( + url, headers=self.generate_headers(), timeout=TIMEOUT_PERIOD + ) + data = json.loads(response.content.decode("utf-8")) + if response.status_code == 201: + return + elif response.status_code == 400: + raise InvalidDomainNameException(data["details"]) + elif response.status_code == 409: + raise DomainConflictException(data["details"]) + + raise Exception("Failed to create protected domain") diff --git a/sdk/setup.py b/sdk/setup.py index a695eb5..ea7dcf7 100644 --- a/sdk/setup.py +++ b/sdk/setup.py @@ -1,10 +1,13 @@ from setuptools import setup, find_packages +import os +TEST_SDK_VERSION = os.getenv("TEST_SDK_VERSION") +version = "0.1.6" setup( name="rapid-sdk", - version="0.1.6", + version=version if TEST_SDK_VERSION is None else f"{version}.{TEST_SDK_VERSION}", description="A python sdk for the rAPId API", - url="https://github.com/no10ds/rapid-sdk", + url="https://github.com/no10ds/rapid/tree/main/sdk", # Originally pointed to a deprecated repo, have updated author="Lewis Card", author_email="lcard@no10.gov.uk", license="MIT", diff --git a/sdk/tests/test_rapid.py b/sdk/tests/test_rapid.py index bd3a267..e95d336 100644 --- a/sdk/tests/test_rapid.py +++ b/sdk/tests/test_rapid.py @@ -18,6 +18,8 @@ InvalidPermissionsException, SubjectNotFoundException, SubjectAlreadyExistsException, + InvalidDomainNameException, + DomainConflictException, ) from .conftest import RAPID_URL, RAPID_TOKEN @@ -392,7 +394,9 @@ def test_delete_client_failure(self, requests_mock: Mocker, rapid: Rapid): rapid.delete_client("xxx-yyy-zzz") @pytest.mark.usefixtures("requests_mock", "rapid") - def update_subject_permissions_success(self, requests_mock: Mocker, rapid: Rapid): + def test_update_subject_permissions_success( + self, requests_mock: Mocker, rapid: Rapid + ): mocked_response = {"data": "dummy"} requests_mock.put( f"{RAPID_URL}/subject/permissions", json=mocked_response, status_code=200 @@ -401,10 +405,58 @@ def update_subject_permissions_success(self, requests_mock: Mocker, rapid: Rapid assert res == mocked_response @pytest.mark.usefixtures("requests_mock", "rapid") - def update_subject_permissions_failure(self, requests_mock: Mocker, rapid: Rapid): + def test_update_subject_permissions_failure( + self, requests_mock: Mocker, rapid: Rapid + ): mocked_response = {"data": "dummy"} requests_mock.put( f"{RAPID_URL}/subject/permissions", json=mocked_response, status_code=400 ) with pytest.raises(InvalidPermissionsException): rapid.update_subject_permissions("xxx-yyy-zzz", ["READ_ALL"]) + + @pytest.mark.usefixtures("requests_mock", "rapid") + def test_create_protected_domain_invalid_name_failure( + self, requests_mock: Mocker, rapid: Rapid + ): + mocked_response = { + "details": "The value set for domain [dummy] can only contain alphanumeric and underscore `_` characters and must start with an alphabetic character" + } + requests_mock.post( + f"{RAPID_URL}/protected_domains/dummy", + json=mocked_response, + status_code=400, + ) + with pytest.raises(InvalidDomainNameException) as exc_info: + rapid.create_protected_domain("dummy") + + assert ( + str(exc_info.value) + == "The value set for domain [dummy] can only contain alphanumeric and underscore `_` characters and must start with an alphabetic character" + ) + + @pytest.mark.usefixtures("requests_mock", "rapid") + def test_create_protected_domain_conflict_failure( + self, requests_mock: Mocker, rapid: Rapid + ): + mocked_response = {"details": "The protected domain, [dummy] already exists"} + requests_mock.post( + f"{RAPID_URL}/protected_domains/dummy", + json=mocked_response, + status_code=409, + ) + with pytest.raises(DomainConflictException) as exc_info: + rapid.create_protected_domain("dummy") + + assert str(exc_info.value) == "The protected domain, [dummy] already exists" + + @pytest.mark.usefixtures("requests_mock", "rapid") + def test_create_protected_domain_success(self, requests_mock: Mocker, rapid: Rapid): + mocked_response = {"data": "dummy"} + requests_mock.post( + f"{RAPID_URL}/protected_domains/dummy", + json=mocked_response, + status_code=201, + ) + res = rapid.create_protected_domain("dummy") + assert res is None diff --git a/ui/package-lock.json b/ui/package-lock.json index ac53ea9..ab991fd 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -16,7 +16,6 @@ "@mui/icons-material": "^5.11.0", "@mui/lab": "^5.0.0-alpha.112", "@mui/material": "^5.10.17", - "@next/font": "^13.0.6", "@tanstack/react-query": "^4.19.1", "@types/react-dom": "18.0.9", "dedent": "^1.5.1", @@ -74,9 +73,9 @@ } }, "node_modules/@adobe/css-tools": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.3.1.tgz", - "integrity": "sha512-/62yikz7NLScCGAAST5SHdnjaDJQBDq0M2muyRTpf2VQhw6StBg2ALiu73zSJQ4fMVLA+0uBhBHAle7Wg+2kSg==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.3.2.tgz", + "integrity": "sha512-DA5a1C0gD/pLOvhv33YMrbf2FK3oUzwNl9oOJqE4XVjuEtt6XIakRcsd7eLiOSPkp1kTRQGICTA8cKra/vFbjw==", "dev": true }, "node_modules/@ampproject/remapping": { @@ -4188,11 +4187,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/@next/font": { - "version": "13.4.19", - "resolved": "https://registry.npmjs.org/@next/font/-/font-13.4.19.tgz", - "integrity": "sha512-yOuSRYfqksWcaG/sATr1/DEGvvI8gnmAAnQCCZ0+L9p4Pio3/DMu71J56YHh9Hz84LDN4tMVzuux0ssCuM50sA==" - }, "node_modules/@next/swc-darwin-arm64": { "version": "13.5.4", "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.5.4.tgz", @@ -12662,20 +12656,23 @@ } }, "node_modules/browserify-sign": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", - "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.2.tgz", + "integrity": "sha512-1rudGyeYY42Dk6texmv7c4VcQ0EsvVbLwZkA+AQB7SxvXxmcD93jcHie8bzecJ+ChDlmAm2Qyu0+Ccg5uhZXCg==", "dev": true, "dependencies": { - "bn.js": "^5.1.1", - "browserify-rsa": "^4.0.1", + "bn.js": "^5.2.1", + "browserify-rsa": "^4.1.0", "create-hash": "^1.2.0", "create-hmac": "^1.1.7", - "elliptic": "^6.5.3", + "elliptic": "^6.5.4", "inherits": "^2.0.4", - "parse-asn1": "^5.1.5", - "readable-stream": "^3.6.0", - "safe-buffer": "^5.2.0" + "parse-asn1": "^5.1.6", + "readable-stream": "^3.6.2", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 4" } }, "node_modules/browserify-sign/node_modules/readable-stream": {