Fix/release process

.env.example

@@ -13,7 +13,6 @@ LAYERS=raw,layer
 RAPID_CLIENT_ID=
 RAPID_CLIENT_SECRET=
 RAPID_URL=
-
 # UI Specific
 NEXT_PUBLIC_API_URL=
 NEXT_PUBLIC_API_URL_PROXY=

.github/workflows/dev.yml

@@ -90,9 +90,15 @@ jobs:
       - name: SDK Test
         run: make sdk-test
 
-      # TODO: Add back in
-      # - name: SDK Test Deploy
-      #   run: make sdk-release-test
+      - name: Create Test Version
+        run: |
+          echo "TEST_SDK_VERSION=$(date +%Y%m%d%H%M%S)" >> $GITHUB_OUTPUT
+        id: version
+
+      - name: SDK Test Deploy
+        run: make sdk-release-test
+        env:
+          TEST_SDK_VERSION: "0.0.0.${{ steps.version.outputs.TEST_SDK_VERSION }}"
 
   ui-dev:
     needs:

.github/workflows/release.yml → .github/workflows/release_api.yml

@@ -6,6 +6,7 @@ on:
 
 jobs:
   setup:
+    if: github.event.release.name == 'API Release'
     runs-on: self-hosted
     steps:
       - name: Checkout
@@ -34,34 +35,6 @@ jobs:
       - name: API Tag and Upload Release Image
         run: make api-tag-and-upload-release-image
 
-  sdk-release:
-    needs:
-      - setup
-    runs-on: self-hosted
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Populate .env with additional vars
-        run: |
-          echo TWINE_USERNAME=${{ secrets.TWINE_USERNAME }} >> .env
-          echo TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} >> .env
-          echo TWINE_NON_INTERACTIVE=true >> .env
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-          cache: 'pip'
-
-      - name: Setup Python Environment
-        run: |
-          make sdk-setup
-          source sdk/.venv/bin/activate
-
-      - name: SDK Release
-        run: make sdk-release
-
   ui-release:
     needs:
       - setup
@@ -91,7 +64,6 @@ jobs:
     needs:
       - setup
       - api-release
-      - sdk-release
       - ui-release
     runs-on: self-hosted
     steps:

.github/workflows/release_sdk.yml

@@ -0,0 +1,57 @@
+name: rAPId Release
+
+on:
+  release:
+    types: [released]
+
+jobs:
+  setup:
+    if: github.event.release.name == 'SDK Release'
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Log commit SHA
+        run: echo $GITHUB_SHA
+
+  sdk-release:
+    needs:
+      - setup
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Populate .env with additional vars
+        run: |
+          echo TWINE_USERNAME=${{ secrets.TWINE_USERNAME }} >> .env
+          echo TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD }} >> .env
+          echo TWINE_NON_INTERACTIVE=true >> .env
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Setup Python Environment
+        run: |
+          make sdk-setup
+          source sdk/.venv/bin/activate
+
+      - name: SDK Release
+        run: make sdk-release
+
+  cleanup:
+    needs:
+      - setup
+      - sdk-release
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Clean Docker Context
+        if: always()
+        run: make clean-pipeline-docker-context

Makefile

@@ -187,19 +187,30 @@ ui-release:
 ui-zip-and-release: ui-zip-contents ui-release ## Zip and release prod static ui site
 
 
-##
-release:
-	@python release.py --operation check
+release-api:
+	@python release.py --operation check --type api
 	@git checkout ${commit}
 	@git tag -a "${version}" -m "Release tag for version ${version}"
 	@git checkout -
 	@git push origin ${version}
-	@python release.py --operation create-changelog
-	@gh release create ${version} -F latest_release_changelog.md
+	@python release.py --operation create-changelog check --type api
+	@gh release create ${version} -F latest_release_changelog_api.md -t "API Release"
 	@rm -rf latest_release_changelog.md
 
+release-sdk:
+	@python release.py --operation check --type sdk
+	@git checkout ${commit}
+	@git tag -a "${version}" -m "Release tag for version ${version}"
+	@git checkout -
+	@git push origin ${version}
+	@python release.py --operation create-changelog check --type sdk
+	@gh release create ${version} -F latest_release_changelog_sdk.md -t "SDK Release"
+	@rm -rf latest_release_changelog.md
 
 # Migration --------------------
 ##
 migrate-v7:			## Run the migration
 	@cd api/; ./batect migrate-v7 -- --layer ${layer} --all-layers ${all-layers}
+
+test-docs:
+	mkdocs serve

docs/changelog.md → docs/changelog/api.md

@@ -1,12 +1,13 @@
+# API Changes
+
 # Changelog
 
-## v7.0.8 / v0.1.6 (sdk) - _2023-11-15_
+## v7.0.8 - _2023-11-15_
 
 ### Fixes
 
 - Issue with date types when editing a schema on the UI because of no option to apply format column and therefore getting an _all fields are required_ error.
 - Tweaked UI design when adding permissions to subject.
-- SDK not uploading a Pandas Dataframe with a date field set correctly.
 - Updated NextJS and Zod package version.
 
 ### Features
@@ -17,11 +18,10 @@
 
 - https://github.com/no10ds/rapid/issues/57
 
-## v7.0.7 / v0.1.5 (sdk) - _2023-11-07_
+## v7.0.7 - _2023-11-07_
 
 ### Fixes
 
-- Issue within the sdk `upload_and_create_dataset` function where schema metadata wasn't being correctly overridden.
 - Hitting maximum security group rules for the load balancer.
 - Documentation improvements and removes any references to the old deprecated repositories.
 
@@ -32,7 +32,7 @@
 - https://github.com/no10ds/rapid/issues/54
 - https://github.com/no10ds/rapid/issues/51
 
-## v7.0.6 / v0.1.4 (sdk) - _2023-10-18_
+## v7.0.6 - _2023-10-18_
 
 ### Features
 
@@ -41,17 +41,10 @@
 
 ### Fixes
 
-- Fixed an issue with the sdk not showing schemas were created successfully due to a wrong response code.
 - Where dataset info was being called on columns with a date type, this was causing an issue with the Pydantic validation.
 - Tweaked the documentation to implement searching for column heading style guide to match what the API returns in the error message.
 
-## v7.0.5 / v0.1.3 (sdk) - _2023-09-20_
-
-### Fixes
-
-- Fix the behaviour of the dataset pattern functions in the SDK.
-
-## v7.0.4 / v0.1.2 (sdk) - _2023-09-20_
+## v7.0.4 - _2023-09-20_
 
 ### Features
 
@@ -63,26 +56,25 @@
 - Updated terraform default `application_version` and `ui_version` variables.
 - Migration script and documentation.
 
-## v7.0.3 / v0.1.2 (sdk) - _2023-09-15_
+## v7.0.3 - _2023-09-15_
 
 ### Fixes
 
 - Fixes issue where permissions were not being correctly read and causing api functionality to fail
 
-## v7.0.2 / v0.1.2 (sdk) - _2023-09-14_
+## v7.0.2 - _2023-09-14_
 
 ### Fixes
 
 - Update UI repo references.
 
-## v7.0.1 / v0.1.2 (sdk) - _2023-09-13_
+## v7.0.1 - _2023-09-13_
 
 ### Fixes
 
 - Date types were being stored as strings which caused issues when querying with Athena. They are now stored as date types.
-- Rename the rAPId sdk method `generate_info` to `fetch_dataset_info` and remove an unnecessary argument.
 
-## v7.0.0 / v0.1.1 (sdk) - _2023-09-12_
+## v7.0.0 - _2023-09-12_
 
 ### Features
 
@@ -96,7 +88,6 @@
 ### Breaking Changes
 
 - All dataset endpoints will be prefixed with `layer`. Typically going from `domain/dataset` to `layer/domain/dataset`.
-- All sdk functions that interact with datasets will now require an argument for layer.
 
 ### Migration
 

docs/changelog/sdk.md

@@ -0,0 +1,75 @@
+# SDK Changes
+
+## v0.1.6 - _2023-11-15_
+
+### Fixes
+
+- SDK not uploading a Pandas Dataframe with a date field set correctly.
+
+### Closes relevant GitHub issues
+
+- https://github.com/no10ds/rapid/issues/57
+
+## v0.1.5  - _2023-11-07_
+
+### Fixes
+
+- Issue within the sdk `upload_and_create_dataset` function where schema metadata wasn't being correctly overridden.
+- Documentation improvements and removes any references to the old deprecated repositories.
+
+
+## v0.1.4 - _2023-10-18_
+
+### Features
+
+- Clients can now be created and deleted via the sdk.
+
+### Fixes
+
+- Fixed an issue with the sdk not showing schemas were created successfully due to a wrong response code.
+- Where dataset info was being called on columns with a date type, this was causing an issue with the Pydantic validation.
+- Tweaked the documentation to implement searching for column heading style guide to match what the API returns in the error message.
+
+## v0.1.3 - _2023-09-20_
+
+### Fixes
+
+- Fix the behaviour of the dataset pattern functions in the SDK.
+
+## v0.1.2 - _2023-09-13_
+
+### Fixes
+
+- Date types were being stored as strings which caused issues when querying with Athena. They are now stored as date types.
+- Rename the rAPId sdk method `generate_info` to `fetch_dataset_info` and remove an unnecessary argument.
+
+## v0.1.1- _2023-09-12_
+
+### Features
+
+- Layers have been introduced to rAPId. These are now the highest level of grouping for your data. They allow you to separate your data into areas that relate to the layers in your data architecture e.g `raw`, `curated`, `presentation`. You will need to specify your layers when you create or migrate a rAPId instance.
+- All the code is now in this monorepo. The previous [Infrastructure](https://github.com/no10ds/rapid-infrastructure), [UI](https://github.com/no10ds/rapid-ui) and [API](https://github.com/no10ds/rapid-api) repos are now deprecated. This will ease the use and development of rAPId.
+- Schemas are now stored in DynamoDB, rather than S3. This offers speed and usability improvements, as well as making rAPId easier to extend.
+- Code efficiency improvements. There were several areas in rAPId where we were executing costly operations that caused performance to degrade at scale. We've fixed these inefficiencies, taking us from O(n²) -> O(n) in these areas.
+- Glue Crawlers have been removed, with Athena tables are created directly by the API instead. Data is now available to query immediately after it is uploaded, rather than the previous wait (approximately 3 mins) while crawlers ran. It also offers scalability benefits because without crawlers we are not dependant on the number of free IPs within the subnet.
+- Improved UI testing with Playwright.
+
+### Breaking Changes
+
+- All dataset endpoints will be prefixed with `layer`. Typically going from `domain/dataset` to `layer/domain/dataset`.
+- All sdk functions that interact with datasets will now require an argument for layer.
+
+### Migration
+
+- See the [migration doc](migration.md) for details on how to migrate to v7 from v6.
+
+[Unreleased changes]: https://github.com/no10ds/rapid/compare/v7.0.8...HEAD
+[v7.0.8 / v0.1.6 (sdk)]: https://github.com/no10ds/rapid/v7.0.7...v7.0.8
+[v7.0.7 / v0.1.5 (sdk)]: https://github.com/no10ds/rapid/v7.0.6...v7.0.7
+[v7.0.6 / v0.1.4 (sdk)]: https://github.com/no10ds/rapid/v7.0.5...v7.0.6
+[v7.0.5 / v0.1.3 (sdk)]: https://github.com/no10ds/rapid/v7.0.4...v7.0.5
+[v7.0.4 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.3...v7.0.4
+[v7.0.3 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.2...v7.0.3
+[v7.0.2 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.1...v7.0.2
+[v7.0.1 / v0.1.2 (sdk)]: https://github.com/no10ds/rapid/v7.0.0...v7.0.1
+[v7.0.0 / v0.1.1 (sdk)]: https://github.com/no10ds/rapid/v7.0.0

mkdocs.yml

@@ -47,10 +47,12 @@ nav:
               - sdk/api/items/schema.md
           - Patterns:
               - sdk/api/patterns/data.md
-  - Releases: changelog.md
+  # - Releases: changelog.md
   - Migration: migration.md
   - Contributing: contributing.md
-
+  - Changelog:
+      - changelog/sdk.md
+      - changelog/api.md
 plugins:
   - search
   - mkdocstrings:

release.py

@@ -2,8 +2,8 @@
 import argparse
 
 
-def create_changelog():
-    with open("./docs/changelog.md", "r") as changelog_file:
+def create_changelog(type):
+    with open("./docs/changelog/" + type + ".md", "r") as changelog_file:
         changelog_lines = changelog_file.readlines()
 
     parsed_lines = []
@@ -25,7 +25,7 @@ def create_changelog():
             "It looks like there is no release information in the changelog. Please check it."
         )
     else:
-        with open("latest_release_changelog.md", "w+") as latest_changelog:
+        with open("latest_release_changelog_" + type + ".md", "w+") as latest_changelog:
             latest_changelog.writelines(parsed_lines)
 
 
@@ -55,9 +55,18 @@ def check():
         required=True,
         choices=["check", "create-changelog"],
     )
+
+    parser.add_argument(
+        "--type",
+        help="What are you releasing?",
+        required=True,
+        choices=["api", "sdk"],
+    )
+
     args = parser.parse_args()
+
     match args.operation:
         case "create-changelog":
-            create_changelog()
+            create_changelog(args.type)
         case "check":
             check()