From e6c0fed39028ac327c776084c8e9d85c0fdd4033 Mon Sep 17 00:00:00 2001 From: Toby Drane Date: Wed, 5 Jul 2023 12:29:18 +0100 Subject: [PATCH] unified changelog and release system --- CHANGELOG.md | 0 LICENSE | 2 +- Makefile | 8 + api/SECURITY.md => SECURITY.md | 0 api/LICENSE | 21 -- api/Makefile | 10 - api/changelog.md | 310 ------------------ api/changelog_release_template.md | 17 - changelog_release_template.md | 61 ++++ ...elog.py => get_latest_release_changelog.py | 2 +- infrastructure/LICENSE | 21 -- infrastructure/Makefile | 8 - infrastructure/SECURITY.md | 59 ---- infrastructure/changelog.md | 189 ----------- infrastructure/changelog_release_template.md | 17 - .../get_latest_release_changelog.py | 22 -- sdk/CHANGELOG.md | 25 -- sdk/LICENSE | 21 -- sdk/Makefile | 9 - sdk/README.md | 45 --- sdk/changelog_release_template.md | 17 - sdk/get_latest_release_changelog.py | 27 -- ui/CHANGELOG.md | 62 ---- ui/CONTRIBUTING.md | 41 --- ui/Makefile | 9 - ui/README.md | 45 +-- ui/changelog_release_template.md | 17 - ui/generate_latest_changelog.js | 33 -- 28 files changed, 74 insertions(+), 1024 deletions(-) create mode 100644 CHANGELOG.md rename api/SECURITY.md => SECURITY.md (100%) delete mode 100644 api/LICENSE delete mode 100644 api/changelog.md delete mode 100644 api/changelog_release_template.md create mode 100644 changelog_release_template.md rename api/get_latest_release_changelog.py => get_latest_release_changelog.py (92%) delete mode 100644 infrastructure/LICENSE delete mode 100644 infrastructure/Makefile delete mode 100644 infrastructure/SECURITY.md delete mode 100644 infrastructure/changelog.md delete mode 100644 infrastructure/changelog_release_template.md delete mode 100644 infrastructure/get_latest_release_changelog.py delete mode 100644 sdk/CHANGELOG.md delete mode 100644 sdk/LICENSE delete mode 100644 sdk/changelog_release_template.md delete mode 100644 sdk/get_latest_release_changelog.py delete mode 100644 ui/CHANGELOG.md delete mode 100644 ui/CONTRIBUTING.md delete mode 100644 ui/changelog_release_template.md delete mode 100755 ui/generate_latest_changelog.js diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/LICENSE b/LICENSE index be77dd3..1add166 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 Number 10 Downing Street +Copyright (c) 2022 Number 10 Downing Street Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 6ad9778..8b2d897 100644 --- a/Makefile +++ b/Makefile @@ -172,3 +172,11 @@ ui-zip-and-release: ## Zip and release prod static ui site @cd ui/; $(MAKE) zip-contents tag=${tag}; $(MAKE) upload-to-release-prod tag=${tag} ## +release: + @git checkout ${commit} + @git tag -a "${version}" -m "Release tag for version ${version}" + @git checkout - + @git push origin ${version} + @python get_latest_release_changelog.py + @gh release create ${version} -F latest_release_changelog.md + @rm -rf latest_release_changelog.md diff --git a/api/SECURITY.md b/SECURITY.md similarity index 100% rename from api/SECURITY.md rename to SECURITY.md diff --git a/api/LICENSE b/api/LICENSE deleted file mode 100644 index 1add166..0000000 --- a/api/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Number 10 Downing Street - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/api/Makefile b/api/Makefile index f539335..4aa6398 100644 --- a/api/Makefile +++ b/api/Makefile @@ -31,16 +31,6 @@ check-app-is-running: @aws ecs wait services-stable --region $(AWS_REGION) --services $(ECS_SERVICE) --cluster $(ECS_CLUSTER) @echo "finished waiting for services to be stable" -# Release ------------------------- -release: - @git checkout ${commit} - @git tag -a "${version}" -m "Release tag for version ${version}" - @git checkout - - @git push origin ${version} - @./batect generate_latest_changelog - @gh release create ${version} -F latest_release_changelog.md - @rm -rf latest_release_changelog.md - tag-release-image: ## Tag the image with the tag name @docker tag rapid-api-service-image:latest $(PUBLIC_URI)/$(PUBLIC_IMAGE):${GITHUB_REF_NAME} diff --git a/api/changelog.md b/api/changelog.md deleted file mode 100644 index ba2b3b0..0000000 --- a/api/changelog.md +++ /dev/null @@ -1,310 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. This project adheres -to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## v6.2.1 - _2023-05-10_ - -See [v6.2.1] changes - -### Fixed - -- An error with the delete dataset endpoint - -## v6.2.0 - _2023-05-10_ - -See [v6.2.0] changes - -### Added - -- The list datasets endpoint now provides the Last Updated Datetime for each dataset - -## v6.1.0 - _2023-05-03_ - -See [v6.1.0] changes - -### Added - -- rAPId can now handle schemas being generated and data being uploaded in Apache Parquet Format. - -### Changed - -- When uploading a file into rAPId we automatically reject any file now that does not match a csv or Apache Parquet format. Before we used to handle the file regardless and wait for a exception being raised from within the api. - -### Fixed - -- When calling the list all datasets endpoint we now filter this list based on the permissions that the user has access to. - - -[v6.1.0]: https://github.com/no10ds/rapid-api/compare/v6.0.2...v6.1.0 - -## v6.0.2 - _2023-03-24_ - -See [v6.0.2] changes - -### Fixed - -- Fixed a bug concerning dataset uploading with overwrite behaviour, making it much less brittle. - -[v6.0.2]: https://github.com/no10ds/rapid-api/compare/v6.0.0...v6.0.2 - -## v6.0.0 - _2023-03-17_ - -See [v6.0.0] changes - -### Fixed - -- All required calls in the API are now paginated by Boto3. This fixes some large issues where, when there were more than 50 crawlers in the account the API would fail to retrieve all datasets as the backend call would paginate onto a next page. -- Fixes an issue where the delete data file endpoint was deleting the raw data file from S3 and now instead deletes the processed file instead. -- Fixes an issue where the uploaded files were temporarily stored with just the name they were uploaded with, this was causing errors if two identically names files were uploaded within a small window. - -### Added - -- New optional environment variable `CATALOG_DISABLED` that can be passed to disable the internal data catalog if required. -- New endpoint that allows for protected domains to be deleted. Can be called using the method `DELETE /api/protected_domains/{domain}`. -- New endpoint that allows for the entire deletion of a dataset from within rAPId. This new method removes all raw and uploaded data files, any schemas, tables and crawlers. Can be thought of an entire dataset wiping from rAPId. The method can be called using `DELETE /api/datasets/{domain}/{dataset}`. - -### Changed - -- __Breaking Change__ - Domains are now case insensitive. This fixes an issue where if you created a Protected domain with an uppercase domain and then the same with a lowercase domain the permissions do not match up as they are interpreted as different endpoints. All domains now have to be lower case. To migrate them, you will need to run: `migrations/scripts/v6_domain_case_insensitive.py`. -- When downloading data the extra Pandas DataFrame index column is not included now. -- FastAPI has been upgraded to 0.92.0. - -### Migration - -To migrate from v5 to v6, you will need to run the migration script: `migrations/scripts/v6_domain_case_insensitive.py`. - -This can be done by first installing the Python requirements from `requirements.txt` and then running `python migrations/scripts/v6_domain_case_insensitive.py` - -You will also need to provide values for the following environment variables, either by defining them in a `.env` file in the repo root or exporting them to the environment where the script is run. - -``` -AWS_REGION -DATA_BUCKET -RESOURCE_NAME_PREFIX -AWS_ACCOUNT_ID -``` - -[v6.0.0]: https://github.com/no10ds/rapid-api/compare/v5.0.2...v6.0.0 - -## v5.0.2 - _2023-03-09_ - -See [v5.0.2] changes - -### Fixed -- Fetching datasets for the UI was only returning write access - -[v5.0.2]: https://github.com/no10ds/rapid-api/compare/v5.0.1...v5.0.2 - -## v5.0.1 - _2023-02-02_ - -See [v5.0.1] changes - -### Fixed -- Fix data always being written to version 1 location - -### Security -- Upgrade GitPython to 3.1.30 - -[v5.0.1]: https://github.com/no10ds/rapid-api/compare/v5.0.0...v5.0.1 - - -## v5.0.0 - _2022-01-27_ - -See [v5.0.0] changes - -v5.0.0 is a major release of rAPId with several breaking changes. One of the major changes is the splitting out of the user interface into it's own codebase. It also introduces a data catalog metadata search. - -### Added - -- New description field for dataset schemas. This is a human readable tag that can be applied to datasets so a user can understand the datasets functionaility. -- Dataset metadata search endpoint `/api/datasets/search/{search term}` - -### Changed -- Test users are created automatically within the infrastructure now and are prefixed accordingly -- ***(Breaking Change)*** All api routes are now served on the prefix `/api`. For instance the api route to list datasets `GET /datasets` now becomes `GET /api/datasets` -- ***(Breaking Change)*** The storage of the pretty printed JSON schemas has been removed to allow for them be queriable within Athena. This is requied for the data catalog search. To migrate over it is required to run the migration `migrations/scripts/v5_schema_migration.py`. -- Test users are now prefixed as they are generated within the infrastructure. - -### Removed - -- The simple static user interface hosted within the API has been removed and is now hosted as it's own (service)[https://github.com/no10ds/rapid-ui] - - -[v5.0.0]: https://github.com/no10ds/rapid-api/compare/v4.2.0...v5.0.0 - -## v4.2.0 - _2022-12-14_ - -See [v4.2.0] changes - -v4.2.0 Improves the behaviour of the query dataset endpoint to allow the querying of large datasets (>100000 rows) - -### Changed - -The query dataset endpoint can now be used for the querying of large datasets (>100000 rows), if the query includes a `limit` clause ensuring that less that 100000 rows of data will be returned. - -[v4.2.0]: https://github.com/no10ds/rapid-api/compare/v4.1.2...v4.2.0 - -## v4.1.2 - _2022-11-29_ - -v4.1.2 Fix for query large dataset client usage - -### Fixes -- Fix query large dataset client usage - -## v4.1.1 - _2022-11-23_ - -v4.1.1 fixes a few issues with the schema creation UI - -### Fixes -- Create client error -- Organisation typo -- Keep sensitivity on second create schema UI page -- Fix owner name validation for create schema - -## v4.1.0 - _2022-11-20_ - -v4.1.0 introduces a new UI for the easy creation of schemas and the ability to expose the rAPId instance to the Gov UK CDDO Federated API Discovery Model (https://github.com/co-cddo/federated-api-model). Instead of having to use the rAPId endpoints to generate and then upload the schema, the new UI opens a simple flow from uploading a file, to altering the generated schema and to then uploading. - -### Added -- UI - - Schema creation flow -- CDDO Federated API Discovery - -## v4.0.0 - _2022-09-21_ - -See [v4.0.0] changes - -v4.0.0 introduces schema versioning for datasets and allows both uploads and downloads. Also, it allows large file -uploads and downloads - -### Added -- Schema versioning - - Schema update endpoint - - Upload and downloading specific version and latest version by default -- Large file upload/download - - Track upload/download job status - - List job status endpoint -- UI - - Task status flow - -### Changed -- Schema versioning - - New schema upload defaulting to version 1 - - Upload/Download data defaults to latest version - -### Security -- Security headers -- Tracing requests by subject ID - -[Unreleased changes]: https://github.com/no10ds/rapid-api/compare/v4.0.0...HEAD -[v4.0.0]: https://github.com/no10ds/rapid-api/compare/v3.0.0...v4.0.0 - -## v3.0.0 - _2022-08-26_ - -See [v3.0.0] changes - -v3.0.0 Provides download functionality in the UI, allowing the user to query. Also, it introduces parquet as the format -to store files, allowing then null values in the datasets. - -### Fixed -- Error when querying files with null values on numeric columns. -- Add protected domains to list all datasets endpoint. - -### Added -- UI - - Input validation on subject creation - - Data management download flow - - Allow user to download datasets from the UI - - Allow user to introduce queries - -### Changed -- Store files in parquet instead of csv -- Data management migrated to parquet - -[v3.0.0]: https://github.com/no10ds/rapid-api/compare/v2.0.0...v3.0.0 - -## v2.0.0 - _2022-08-19_ - -See [v2.0.0] changes - -v2.0.0 provides a complete overhaul on how we handle authorisation as well as an extended UI. - -### Fixed -- Consistent exception handling - -### Added -- Endpoints: - - Create subject - - List subjects - - Delete subject - - Modify subject permissions - - Get subject permissions - - Get all permissions -- UI - - User management flows - - Data management flows -- UI user journey test - -### Changed -- Complete overhaul of authorisation process - -[v2.0.0]: https://github.com/no10ds/rapid-api/compare/v1.3.0...v2.0.0 - -## v1.3.0 - _2022-06-17_ - -See [v1.3.0] changes - -### Added -- Getting started documentation to walkthrough dataset uploading and querying - -### Fixed -- Add `resource_prefix` to the necessary AWS resources to enable the hosting of multiple rAPId instances - -[v1.3.0]: https://github.com/no10ds/rapid-api/compare/v1.2.0...v1.3.0 - -## v1.2.0 - _2022-05-31_ - -See [v1.2.0] changes - -### Added -- Documentation improvements - - OpenAPI spec includes endpoint behaviour documentation - - Added example scripts for programmatic interaction - -[v1.2.0]: https://github.com/no10ds/rapid-api/compare/v1.1.0...v1.2.0 - -## v1.1.0 - _2022-05-29_ - -See [v1.1.0] changes - -### Added -- Protected domains: - - Allows the separation of access permissions for specific `protected domains` - - See the data acccess [docs](./docs/guides/usage/data_access.md) - - See usage [docs](./docs/guides/usage/usage.md#domain) -- Overwrite update behaviour: - - Allows datasets to be overwritten when a new file is uploaded, rather than just appended to - - See the schema creation [docs](./docs/guides/usage/schema_creation.md#update-behaviour) - -[v1.1.0]: https://github.com/no10ds/rapid-api/compare/v1.0.0...v1.1.0 - - -## v1.0.0 - _2022-05-10_ - -See [v1.0.0] changes - -### Added -- Documentation and usage guides can be found [here](https://github.com/no10ds/rapid-api/tree/master/docs) -- First full application release -- Features: - - Generate a schema from a dataset - - Upload a schema - - Upload a dataset - - List available datasets - - Get metadata information for a dataset - - Query a previously uploaded dataset - - Add a new client app - -[Unreleased changes]: https://github.com/no10ds/rapid-api/compare/v1.0.0...HEAD -[v1.0.0]: https://github.com/no10ds/rapid-api/compare/ff60bf65...v1.0.0 diff --git a/api/changelog_release_template.md b/api/changelog_release_template.md deleted file mode 100644 index 88f1ed0..0000000 --- a/api/changelog_release_template.md +++ /dev/null @@ -1,17 +0,0 @@ -## vX.X.X - _YYYY-MM-DD_ - -See [vX.X.X] changes - -### Fixed - -### Added - -### Changed - -### Removed - -### Deprecated - -### Security - -[vX.X.X]: https://github.com/no10ds/rapid-api/compare/...vX.X.X diff --git a/changelog_release_template.md b/changelog_release_template.md new file mode 100644 index 0000000..708210f --- /dev/null +++ b/changelog_release_template.md @@ -0,0 +1,61 @@ +## vX.X.X - _YYYY-MM-DD_ + +See [vX.X.X] changes + +### API + +#### Fixed + +#### Added + +#### Changed + +#### Removed + +#### Deprecated + +#### Security + +### Infrastructure + +#### Fixed + +#### Added + +#### Changed + +#### Removed + +#### Deprecated + +#### Security + +### SDK + +#### Fixed + +#### Added + +#### Changed + +#### Removed + +#### Deprecated + +#### Security + +### UI + +#### Fixed + +#### Added + +#### Changed + +#### Removed + +#### Deprecated + +#### Security + +[vX.X.X]: https://github.com/no10ds/rapid/compare/...vX.X.X diff --git a/api/get_latest_release_changelog.py b/get_latest_release_changelog.py similarity index 92% rename from api/get_latest_release_changelog.py rename to get_latest_release_changelog.py index eb12778..4e6a796 100644 --- a/api/get_latest_release_changelog.py +++ b/get_latest_release_changelog.py @@ -1,6 +1,6 @@ import re -with open("changelog.md", "r") as changelog_file: +with open("CHANGELOG.md", "r") as changelog_file: changelog_lines = changelog_file.readlines() parsed_lines = [] diff --git a/infrastructure/LICENSE b/infrastructure/LICENSE deleted file mode 100644 index 1add166..0000000 --- a/infrastructure/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Number 10 Downing Street - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/infrastructure/Makefile b/infrastructure/Makefile deleted file mode 100644 index b5ee813..0000000 --- a/infrastructure/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -release: ## Release - @git checkout ${commit} - @git tag -a "${version}" -m "Release tag for version ${version}" - @git checkout - - @git push origin ${version} - @./batect generate_latest_changelog - @gh release create ${version} -F latest_release_changelog.md - @rm -rf latest_release_changelog.md diff --git a/infrastructure/SECURITY.md b/infrastructure/SECURITY.md deleted file mode 100644 index 30b0532..0000000 --- a/infrastructure/SECURITY.md +++ /dev/null @@ -1,59 +0,0 @@ -# Security Notice - -## What? - -This is the security notice for all [No10 Data Science repositories](https://github.com/No10ds). The notice explains how vulnerabilities should be reported. Within Cabinet Office there are cyber security and information assurance teams, as well as security-conscious people within the programmes, that assess and triage all reported vulnerabilities. - -## Reporting a Vulnerability - -Cabinet Office and No10 are advocates of responsible vulnerability disclosure. If you’ve found a vulnerability, we would like to know so we can fix it. This notice provides details for how you can let us know about vulnerabilities, additionally you can view our [security.txt](https://vdp.cabinetoffice.gov.uk/.well-known/security.txt) file, here: - -You can report a vulnerability through our vulnerability disclosure programme at [HackerOne](https://hackerone.com/44c348eb-e030-4273-b445-d4a2f6f83ba8/embedded_submissions/new). Alternatively, you can send an email to [disclosure@digital.cabinet-office.gov.uk](mailto:disclosure@digital.cabinet-office.gov.uk); if you do this you may get a response from Zendesk, which is our ticketing system. - -When reporting a vulnerability to us, please include: - -- the website, page or repository where the vulnerability can be observed -- a brief description of the vulnerability -- details of the steps we need to take to reproduce the vulnerability -- non-destructive exploitation details - -If you are able to, please also include: - -- the type of vulnerability, for example, the [OWASP category](https://owasp.org/www-community/vulnerabilities/) -- screenshots or logs showing the exploitation of the vulnerability - -[Reach out via email](mailto:disclosure@digital.cabinet-office.gov.uk) if you are not sure if the vulnerability is genuine and exploitable, or you have found: - -- a non-exploitable vulnerability -- something you think could be improved - for example, missing security headers -- TLS configuration weaknesses - for example weak cipher suite support or the presence of TLS1.0 support - -## Guidelines for reporting a vulnerability - -When you are investigating and reporting the vulnerability on a gov.uk domain or subdomain, you must not: - -- break the law -- access unnecessary or excessive amounts of data -- modify data -- use high-intensity invasive or destructive scanning tools to find vulnerabilities -- try a denial of service - for example overwhelming a service with a high volume of requests -- disrupt our services or systems -- tell other people about the vulnerability you have found until we have disclosed it -- social engineer, phish or physically attack our staff or infrastructure -- demand money to disclose a vulnerability - -Only submit reports about exploitable vulnerabilities through HackerOne. - -## Bug bounty - -Unfortunately, Cabinet Office and No10 do not offer a paid bug bounty programme. We will make efforts to show appreciation to people who take the time and effort to disclose vulnerabilities responsibly. We do have [an acknowledgements page for legitimate issues found by researchers](https://vdp.cabinetoffice.gov.uk/thanks.txt). - ---- - -#### Further reading and inspiration about responsible disclosure and `SECURITY.md` - -- -- -- -- -- diff --git a/infrastructure/changelog.md b/infrastructure/changelog.md deleted file mode 100644 index 2ba9d75..0000000 --- a/infrastructure/changelog.md +++ /dev/null @@ -1,189 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. This project adheres -to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## v6.2.1 - _2023-06-29_ - -See [v6.2.1] changes - -### Fixed - -- Update UI bucket ownership controls to set ACL. Due to AWS change. - - -## v6.2.0 - _2023-05-10_ - -See [v6.2.0] changes - -### Fixed - -- Password policy rules can now be passed through to all modules in the iam-config block. - -## v6.1.0 - _2023-05-03_ - -See [v6.1.0] changes - -### Fixed - -- We now apply a WAF oversize handling rule directly through the infrastructure as per https://github.com/hashicorp/terraform-provider-aws/issues/25545. -- When deploying multiple instances of rAPId on a same account but to different environments e.g. dev or prod, some of the infrastructure had conflicting names and therefore would fail to deploy. -- Limit access policy further for the access logs iam policy to pass Checkov scanning. - - -## v6.0.3 - _2023-04-12_ - -See [v6.0.3] changes - -### Fixed -- Add Glue:GetDatabases permission to ECS task role. -- Make the UI rebuild trigger if the S3 bucket changes. - -## v6.0.2 - _2023-03-28_ - -See [v6.0.2] changes - -### Changed -- Limit the size of the random string added to the S3 bucket, due to the 63 character bucket name limit. - -## v6.0.1 - _2023-03-21_ - -See [v6.0.1] changes - -### Changed -- Updated the default values for ui_version and application_version to v6.0.1 - -## v6.0.0 - _2023-03-17_ - -See [v6.0.0] changes - -### Changed -- All terraform providers have been updated and are now consistent across the infrastructure. - -### Added -- New variable to be passed into the `app-cluster` if you want to disable the internal API data catalog. Defaults to false but can be set as true. - -## v5.0.2 - _2023-03-07_ -See [v5.0.2] changes - -### Fixed -- Stop cerfiticate validation record duplication - -## v5.0.1 - _2023-02-01_ -See [v5.0.1] changes - -### Fixed -- Removed the requirement to inject a API into the URL - -## v5.0.0 - _2023-02-01_ -See [v5.0.0] changes - -### Added -- Infrastructure required to deploy new UI. It introduces two new variables. `ui_version=v5.0.0` the version of the static UI to download and `us_east_certificate_validation_arn` a new arn required to be created in `us-east-1` for Cloudfront. The following get's created - - New S3 bucket to host static html files - - Cloudfront hosting the API in ECS and the S3 static html files - - Lambda edge function required for the html page routing -- Test clients are now created directly within the infrastructure - -### Fixed -- Prevents ECS load balancer from getting deleted -- Removed inline policies in use -- Glue is allowed the PassRole access to allow for schema uploading -- S3 buckets now have SSL restrictions - -## v4.2.0 - _2022-12-22_ -See [v4.2.0] changes - -### Added -- Configurable cognito password policy -- Cloudtrail creation is now optional and cloudtrail events are now consolidated into one trail - -## v4.1.1 - _2022-11-23_ -See [v4.1.1] changes - -### Fixes -- Typo in organisation - -## v4.1.0 - _2022-11-20_ -See [v4.1.0] changes - -### Added -- Read in optional project details for CDDo federated API model -- New global secondary index column for jobs DynamoDB table - -## v4.0.0 - _2022-09-21_ -See [v4.0.0] changes - -### Fixed -- Query results expiry time - -### Added -- DynamoDB resource for job status management - -## v3.0.0 - _2022-08-26_ - -See [v3.0.0] changes - -### Fixed -- Oversized requests handling - -### Added -- Lifecycle policy to ECR - -### Changed -- Allow service task to delete parquet files - -### Removed -- Custom csv classifier - -## v2.0.0 - _2022-08-19_ - -See [v2.0.0] changes - -### Added -- DynamoDB resources for new authorisation structure - -### Removed -- Custom permission scopes - -### Security -- Handle oversized requests in WAF manually until terraform is updated - -## v1.3.0 - _2022-07-25_ - -See [v1.3.0] changes - -### Added -- Documentation and usage guides can be found [here](https://github.com/no10ds/rapid-infrastructure/tree/master/docs) -- First full application release -- Features: - - Build rapid infrastructure - - Add release mechanism see [release](https://github.com/no10ds/rapid-infrastructure/blob/main/docs/guides/contributing.md#releasing) - -## v1.0.0 - _2022-07-20_ - -See [v1.0.0] changes - -### Added -- Documentation and usage guides can be found [here](https://github.com/no10ds/rapid-infrastructure/tree/master/docs) -- First full application release -- Features: - - Build rapid infrastructure - - -[v6.1.0]: https://github.com/no10ds/rapid-infrastructure/compare/v6.0.3...HEAD -[v6.0.3]: https://github.com/no10ds/rapid-infrastructure/compare/v6.0.2...v6.0.3 -[v6.0.2]: https://github.com/no10ds/rapid-infrastructure/compare/v6.0.1...v6.0.2 -[v6.0.1]: https://github.com/no10ds/rapid-infrastructure/compare/v6.0.0...v6.0.1 -[v6.0.0]: https://github.com/no10ds/rapid-infrastructure/compare/v5.0.2...v6.0.0 -[v5.0.2]: https://github.com/no10ds/rapid-infrastructure/compare/v5.0.1...v5.0.2 -[v5.0.1]: https://github.com/no10ds/rapid-infrastructure/compare/v5.0.0...v5.0.1 -[v5.0.0]: https://github.com/no10ds/rapid-infrastructure/compare/v4.2.0...v5.0.0 -[v4.2.0]: https://github.com/no10ds/rapid-infrastructure/compare/v4.1.0...v4.2.0 -[v4.1.1]: https://github.com/no10ds/rapid-infrastructure/compare/v4.1.0...v4.1.1 -[v4.1.0]: https://github.com/no10ds/rapid-infrastructure/compare/v4.0.0...v4.1.0 -[v4.0.0]: https://github.com/no10ds/rapid-infrastructure/compare/v3.0.0...v4.0.0 -[v3.0.0]: https://github.com/no10ds/rapid-infrastructure/compare/v2.0.0...v3.0.0 -[v2.0.0]: https://github.com/no10ds/rapid-infrastructure/compare/v1.3.0...v2.0.0 -[v1.3.0]: https://github.com/no10ds/rapid-infrastructure/compare/v1.0.0...v1.3.0 -[v1.0.0]: https://github.com/no10ds/rapid-infrastructure/compare/5298389...v1.0.0 diff --git a/infrastructure/changelog_release_template.md b/infrastructure/changelog_release_template.md deleted file mode 100644 index 3e4e8bd..0000000 --- a/infrastructure/changelog_release_template.md +++ /dev/null @@ -1,17 +0,0 @@ -## vX.X.X - _YYYY-MM-DD_ - -See [vX.X.X] changes - -### Fixed - -### Added - -### Changed - -### Removed - -### Deprecated - -### Security - -[vX.X.X]: https://github.com/no10ds/rapid-infrastructure/compare/...HEAD diff --git a/infrastructure/get_latest_release_changelog.py b/infrastructure/get_latest_release_changelog.py deleted file mode 100644 index 1f10e7e..0000000 --- a/infrastructure/get_latest_release_changelog.py +++ /dev/null @@ -1,22 +0,0 @@ -import re - -with open("changelog.md", "r") as changelog_file: - changelog_lines = changelog_file.readlines() - -parsed_lines = [] - -adding = False - -for line in changelog_lines: - if re.match(f"##\sv\d+\.\d+\.\d+\s-\s_\d{{4}}-\d{{2}}-\d{{2}}_$", line): - adding = not adding - if adding: - parsed_lines.append(line) - -if not parsed_lines: - raise Exception( - "It looks like there is no release information in the changelog. Please check it." - ) -else: - with open("latest_release_changelog.md", "w+") as latest_changelog: - latest_changelog.writelines(parsed_lines[4:]) diff --git a/sdk/CHANGELOG.md b/sdk/CHANGELOG.md deleted file mode 100644 index 3622330..0000000 --- a/sdk/CHANGELOG.md +++ /dev/null @@ -1,25 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. This project adheres -to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## v0.0.8 - _2023-03-07_ - -See [v0.0.8] changes - -### Changed -- Introduced new Pydantic class types for all `rapid.items`. This allows for type validation on the class creation and matches the types within the rAPId to help minimize errors. -- `rapid.Rapid.generate_schema()` now returns the new Pydantic `rapid.items.Schema` class. -- `rapid.Rapid.patterns` now includes common patterns that might be used for data manipulation tasks exploiting the sdk functions for easier use. -- Improved documentation across the entire sdk. - -### Removed -- `create` and `update` methods are removed from the old `rapid.items.Schema` class as it didn't make sense to have api calls within classes. - -### Added -- New `create_schema` and `update_schema` methods added to the `rapid.Rapid` class. This is to mock the behaviour mentioned above. -- New `rapid.items.Query` Pydantic class that allows for a programmatic definition of a rAPId dataset download. -- New `rapid.Rapid.download_dataset` function. Returns the downloaded Pandas dataframe. This function makes use of the above `Query` class. -- Improved unit testing coverage. - -[v0.0.8]: https://github.com/no10ds/rapid-api/compare/720ecd1fd6ce14fc37a3202133ce239419d48f1f...v0.0.8 diff --git a/sdk/LICENSE b/sdk/LICENSE deleted file mode 100644 index 1add166..0000000 --- a/sdk/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Number 10 Downing Street - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/sdk/Makefile b/sdk/Makefile index 1052f71..54eb19c 100644 --- a/sdk/Makefile +++ b/sdk/Makefile @@ -14,15 +14,6 @@ reqs: @echo "Virtual environment successfully created. To activate the venv:" @echo " \033[0;32msource sdk/.venv/bin/activate" -release: - git checkout ${commit} - git tag -a "${version}" -m "Release tag for version ${version}" - git checkout - - git push origin ${version} - python get_latest_release_changelog.py - @gh release create ${version} -F latest_release_changelog.md - rm -rf latest_release_changelog.md - cleanup: rm -rf ./dist diff --git a/sdk/README.md b/sdk/README.md index 9be1f55..0ed6fb7 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -1,51 +1,6 @@

rAPId-sdk

-

- - - - -Package version - -

- # About Please see the [docs page](https://rapid-sdk.readthedocs.io/en/latest/) for greater reference. rAPId-sdk is a Python wrapper for the rapid-api repository. It provides a simple and intuitive interface for accessing the functionality of the rapid-api library in Python. - -# Contributing - -To install rAPId sdk, you will need to have pyenv and python installed. You can use the provided Makefile to set up the required dependencies and create a virtual environment for the project: - -``` -make setup -``` - -This will install pyenv and the specified version of python, create a virtual environment for the project, and install the required dependencies. - -To create a test deployment and upload the package to testpypi run `make deploy/test` - -# Releasing - -To perform a new release of the sdk see the guide below. - -## Context - -Performing a release involves tagging a new version of the sdk with the specific verison number. - -## Steps - -1. Decide on the new version number following the [semantic versioning approach](https://semver.org/) -2. Update and commit the [Changelog](./CHANGELOG.md) (you can follow the [template](./changelog_release_template.md)) -3. Update the version number in `setup.py` and commit this. -4. Finally update the ReadTheDocs contents by running `make documentation/build` and then commit this. -4. Run `make release commit= version=vX.X.X` - -> ⚠️ Ensure the version number follows the format `vX.X.X` with full-stops in the same places. - -Now the release pipeline will run automatically and build the sdk off that version of the code and finally push it to PyPi. - -# License - -rAPId api is released under the MIT License. diff --git a/sdk/changelog_release_template.md b/sdk/changelog_release_template.md deleted file mode 100644 index 09ac338..0000000 --- a/sdk/changelog_release_template.md +++ /dev/null @@ -1,17 +0,0 @@ -## vX.X.X - _YYYY-MM-DD_ - -See [vX.X.X] changes - -### Fixed - -### Added - -### Changed - -### Removed - -### Deprecated - -### Security - -[vX.X.X]: https://github.com/no10ds/rapid-sdk/compare/...vX.X.X diff --git a/sdk/get_latest_release_changelog.py b/sdk/get_latest_release_changelog.py deleted file mode 100644 index 768ace4..0000000 --- a/sdk/get_latest_release_changelog.py +++ /dev/null @@ -1,27 +0,0 @@ -import re - -with open("CHANGELOG.md", "r", encoding="utf-8") as changelog_file: - changelog_lines = changelog_file.readlines() - -parsed_lines = [] - -adding = False - -for line in changelog_lines: - if re.match( - f"##\\sv\\d+\\.\\d+\\.\\d+\\s-\\s_\\d{{4}}-\\d{{2}}-\\d{{2}}_$", # noqa: F541 - line, - ): - adding = not adding - if adding: - parsed_lines.append(line) - -if not parsed_lines: - raise Exception( - "It looks like there is no release information in the changelog. Please check it." - ) -else: - with open( - "latest_release_changelog.md", "w+", encoding="utf-8" - ) as latest_changelog: - latest_changelog.writelines(parsed_lines[4:]) diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md deleted file mode 100644 index 6aa75fb..0000000 --- a/ui/CHANGELOG.md +++ /dev/null @@ -1,62 +0,0 @@ -# Changelog - -## v6.2.0 - _2023-03-21_ - -See [v6.2.0] changes - -### Changed - -- The upload data page now tracks the upload progress - -## v6.0.1 - _2023-03-21_ - -See [v6.0.1] changes - -### Fixed - -- The delete data option is now only displayed to users with the DATA_ADMIN permission. - -## v6.0.0 - _2023-03-17_ - -See [v6.0.0] changes - -### Added -- New UI endpoint for deleting an entire dataset. - -### Changed -- When the API does not include the internal data catalog we do not show the catalog option on the sidebar. -- When downloading data with multiple versions we now default to the most recent version instead of the first version. -- When uploading data we change the upload complete alert from a success to an info. This nudges the user more to release that the data is still processing. - -### Fixed -- Handles case when no data is present within rAPId. Before the UI would crash. - -## v5.0.2 - _2023-03-09_ - -See [v5.0.2] changes - -### Fixed - -- Fetch available datasets separately for both Read and Write access - -## v5.0.1 - _2023-01-02_ - -See [v5.0.1] changes - -### Fixed - -- Removed the need to have API_URL environment variable. Fixes issue with this sometimes being undefined. - -## v5.0.0 - _2023-01-24_ - -See [v5.0.0] changes - -### Added -- Initial release of new rAPId UI - -[v6.2.0]: https://github.com/no10ds/rapid-ui/compare/v6.0.1...v6.2.0 -[v6.0.1]: https://github.com/no10ds/rapid-ui/compare/v6.0.0...v6.0.1 -[v6.0.0]: https://github.com/no10ds/rapid-ui/compare/v5.0.2...v6.0.0 -[v5.0.2]: https://github.com/no10ds/rapid-ui/comapre/v5.0.1...v5.0.2 -[v5.0.1]: https://github.com/no10ds/rapid-ui/compare/v5.0.0...v5.0.1 -[v5.0.0]: https://github.com/no10ds/rapid-ui/compare/...v5.0.0 diff --git a/ui/CONTRIBUTING.md b/ui/CONTRIBUTING.md deleted file mode 100644 index dda7599..0000000 --- a/ui/CONTRIBUTING.md +++ /dev/null @@ -1,41 +0,0 @@ -# Contributing - -## Developing 🤓 - -To maintain a consistent version of node across machines we use node version manager. Once this is installed you can install the correct version of node for this project using the command - -``` -nvm use -``` - -By default the application checks for a valid rAPId access token (rat) cookie, if this is not available you will be redirected to the login page. When developing locally to bypass this you need to generate a valid rat token and place this as a cookie in both the frontend and in the api. - -## Testing - -By default precommit hooks are setup using husky to automatically lint the project and run the ui unit tests. - -To run them all you can use the command `make test` - -## Releasing - -### Context - -The product of the rAPId UI is a set of complete static HTML files that using the provided [infrastructure](https://github.com/no10ds/rapid-infrastructure) can be deployed and served alongside the API. - -Performing a release requires tagging the repository with the specific version and then building a zip of the static html files that can be served within a GitHub release. - -⚠️ When releasing a new version of the service, you must also release a version of -the [api](https://github.com/no10ds/rapid-api) [infrastructure](https://github.com/no10ds/rapid-infrastructure) and vice versa. All versions should be the same ( -i.e.: all vX.Y.Z). If there are no changes in one or the others repo, it should still be released along with the other -and a tag added to the same commit as the previous release. This ensures that the version numbers signal the -compatibility between all the elements. - -### Steps - -1. Decide on the new version number following the [semantic versioning approach](https://semver.org) -2. Update and commit the [Changelog](CHANGELOG.md) (you can follow the [template](changelog_release_template.md)) -3. Run `make release commit= version=vX.X.X` - -> ⚠️ Ensure the version number follows the format `vX.X.X` with full-stops in the same places. - -Now the release pipeline will run automatically, this will build the static html files, zip them and upload them to the release tag in GitHub. diff --git a/ui/Makefile b/ui/Makefile index 6c5a027..51abbc2 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -31,12 +31,3 @@ upload-to-release-prod: ## Upload the zipped built static files to a production create-static-out: ## Manually create the static files @npm run build:static - -release: - @git checkout ${commit} - @git tag -a "${version}" -m "Release tag for version ${version}" - @git checkout - - @git push origin ${version} - @node ./generate_latest_changelog.js - @gh release create ${version} -F latest_release_changelog.md - @rm -rf latest_release_changelog.md diff --git a/ui/README.md b/ui/README.md index b420187..3746538 100644 --- a/ui/README.md +++ b/ui/README.md @@ -1,50 +1,11 @@ -# rAPId UI - - - -The rAPId service allows users to ingest, validate and query data via an API. This repo provides the user interface service for rAPId version >= 5.0 - -
-

- -

+

rAPId-ui

# About -Since rAPId version 5.0 the user interface to interact with the API was seperated into it's own service. The user interface is compiled and built to static html files and hosted through a Cloud Delivery Network (CDN). +The rAPId user interface is compiled and built to static html files and hosted through a Cloud Delivery Network (CDN). It allows for a user friendly wrapper around rAPId without building the logic directly into the api. -# Tech Stack 🍭 +# Tech Stack - Typescript - NodeJs - NextJs - -# Developing - -This is a quick guide to running the rAPId UI locally for development. For greater details please see the [Contributing README](CONTRIBUTING.md) - -## Prerequisites - -Install all the required tools - -- [Node Version Manager (nvm)](https://github.com/nvm-sh/nvm#installing-and-updating) -- Install and use the required version of node `nvm use` - -Install the packages and pre-commit hooks. - -1. Install packages & husky `make npm-setup` - -## Running Locally 🏃‍♂️ - -Copy over the `.env.example` file into a seperate `.env.local` file and then populate the environment variable to point the api url to your current running rAPId api. To get authentication working you will need the relevant `rat` cookie stored in the browser. - -To run the app locally you can then run `make dev`. - -## Local environment variables - -Use the below to proxy to a third party auth server - -``` -NEXT_PUBLIC_API_URL=/myapi -NEXT_PUBLIC_API_URL_PROXY=https://some-apiurl-from-another-server #optional for local development only -``` diff --git a/ui/changelog_release_template.md b/ui/changelog_release_template.md deleted file mode 100644 index c30e9c7..0000000 --- a/ui/changelog_release_template.md +++ /dev/null @@ -1,17 +0,0 @@ -## vX.X.X - _YYYY-MM-DD_ - -See [vX.X.X] changes - -### Fixed - -### Added - -### Changed - -### Removed - -### Deprecated - -### Security - -[vX.X.X]: https://github.com/no10ds/rapid-ui/compare/...HEAD diff --git a/ui/generate_latest_changelog.js b/ui/generate_latest_changelog.js deleted file mode 100755 index d9a462c..0000000 --- a/ui/generate_latest_changelog.js +++ /dev/null @@ -1,33 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires -const fs = require('fs') - -const file = fs.readFileSync('./CHANGELOG.md', { encoding: 'utf-8' }) - -const parsedLines = [] -let adding = false - -file.split(/\r?\n/).forEach((line) => { - const pattern = /## v[0-9]\.[0-9]\.[0-9] - _[0-9]{4}-[0-9]{2}-[0-9]{2}_/ - if (pattern.test(line)) { - adding = !adding - } - - if (adding) { - parsedLines.push(line) - } -}) - -if (!parsedLines.length) { - throw new Error( - 'It looks like there is no release information in the changelog. Please check it.' - ) -} else { - const file = fs.createWriteStream('latest_release_changelog.md') - file.on('error', (error) => { - throw new Error(error) - }) - parsedLines.forEach((line) => { - file.write(`${line} \n`) - }) - file.end() -}