Fix/actions

.env.example

@@ -1,12 +1,13 @@
-AWS_ACCOUNT=
-AWS_REGION=
+AWS_ACCOUNT=123456
+AWS_REGION=eu-west-2
 
 # API Specific
-COGNITO_USER_POOL_ID=rapid-pool
-DATA_BUCKET=rapid-bucket
-DOMAIN_NAME=rapid-domain
-RESOURCE_PREFIX=rapid
 ALLOWED_EMAIL_DOMAINS=example1.com,example2.com
+DATA_BUCKET=the-bucket
+RESOURCE_PREFIX=rapid
+DOMAIN_NAME=example.com
+COGNITO_USER_POOL_ID=11111111
+LAYERS=raw,layer
 
 # SDK Specific
 RAPID_CLIENT_ID=

.github/.github.env

@@ -2,3 +2,5 @@ COGNITO_USER_POOL_ID=rapid-pool
 RESOURCE_PREFIX=rapid
 ALLOWED_EMAIL_DOMAINS=example1.com,example2.com
 LAYERS=raw,layer
+DOMAIN_NAME=example.com
+DATA_BUCKET=the-bucket

.github/workflows/dev.yml

@@ -45,8 +45,6 @@ jobs:
       - name: Populate .env with additional vars
         run: |
           cp ./.github/.github.env .env
-          echo DOMAIN_NAME=${{ secrets.DOMAIN_NAME }} >> .env
-          echo DATA_BUCKET=${{ secrets.DATA_BUCKET }} >> .env
           echo AWS_ACCOUNT=${{ secrets.AWS_ACCOUNT }} >> .env
           echo AWS_REGION=${{ secrets.AWS_REGION }} >> .env
           echo AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} >> .env
@@ -76,7 +74,7 @@ jobs:
         run: |
           echo "TWINE_USERNAME=${{ secrets.TWINE_USERNAME_TEST }}" >> .env
           echo "TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD_TEST }}" >> .env
-          echo TWINE_NON_INTERACTIVE=${{ secrets.TWINE_NON_INTERACTIVE }} >> .env
+          echo "TWINE_NON_INTERACTIVE=true" >> .env
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/main.yml

@@ -44,8 +44,6 @@ jobs:
       - name: Populate .env with additional vars
         run: |
           cp ./.github/.github.env .env
-          echo DOMAIN_NAME=${{ secrets.DOMAIN_NAME }} >> .env
-          echo DATA_BUCKET=${{ secrets.DATA_BUCKET }} >> .env
           echo AWS_ACCOUNT=${{ secrets.AWS_ACCOUNT }} >> .env
           echo AWS_REGION=${{ secrets.AWS_REGION }} >> .env
           echo AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} >> .env
@@ -69,7 +67,7 @@ jobs:
         run: make api-tag-prod-candidate
 
       - name: API Deploy Image to Prod
-        run: make api-tag-live-in-prod
+        run: make api-app-live-in-prod
 
       - name: API Allow for Application to Start
         run: sleep 120
@@ -81,6 +79,8 @@ jobs:
       - name: API E2E Tests
         id: e2e-tests
         env:
+          DOMAIN_NAME: ${{ secrets.DOMAIN_NAME }}
+          DATA_BUCKET: ${{ secrets.DATA_BUCKET }}
           COGNITO_USER_POOL_ID: ${{ secrets.COGNITO_USER_POOL_ID }}
           RESOURCE_PREFIX: ${{ secrets.RESOURCE_PREFIX }}
           ALLOWED_EMAIL_DOMAINS: ${{ secrets.ALLOWED_EMAIL_DOMAINS }}

.github/workflows/ui-tests.yml

@@ -1,18 +1,13 @@
 name: rAPId Integration Tests
 
 on:
-  push:
-    branches:
-      - "**"
-
   workflow_dispatch:
 
   pull_request:
     types:
       - opened
 
 jobs:
-
   run-ui-test:
     runs-on: self-hosted
 
@@ -29,11 +24,13 @@ jobs:
           npm install
 
       - name: Install playwright browsers
-        run: npx playwright install-deps && npx playwright install
+        run: |
+          cd ui
+          npm install @playwright/test -D
 
-      - name: run playwright tests
-        run: npx playwright test ui/playwright
+      - name: Run playwright tests
+        run: make ui-test-e2e
         env:
-          DOMAIN: ${{ secrets.DOMAIN }}
+          DOMAIN_NAME: "https://${{ secrets.DOMAIN_NAME }}"
           RESOURCE_PREFIX: ${{ secrets.RESOURCE_PREFIX }}
           AWS_REGION: ${{ secrets.AWS_REGION }}

.gitignore

@@ -182,5 +182,6 @@ docs/_build/
 .terraform/
 .terraform.lock.hcl
 
-playwright/.auth
-playwright/.downloads
+ui/playwright/.auth
+ui/playwright/.downloads
+ui/test-results/

.pre-commit-config.yaml

@@ -1,70 +1,75 @@
 exclude: '^ui'
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.3.0
     hooks:
-    -   id: check-yaml
-    -   id: check-json
-    -   id: check-merge-conflict
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
--   repo: https://github.com/PyCQA/bandit
+      - id: check-yaml
+      - id: check-json
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/PyCQA/bandit
     rev: 1.7.5
     hooks:
-    -   id: bandit
+      - id: bandit
         exclude: '(test|docs)/*'
--   repo: https://github.com/psf/black
+  - repo: https://github.com/psf/black
     rev: 22.6.0
     hooks:
-    -   id: black
--   repo: https://github.com/Yelp/detect-secrets
+      - id: black
+  - repo: https://github.com/Yelp/detect-secrets
     rev: v1.3.0
     hooks:
-    -   id: detect-secrets
+      - id: detect-secrets
         exclude: docs/
--   repo: https://github.com/asottile/blacken-docs
+  - repo: https://github.com/asottile/blacken-docs
     rev: v1.12.1
     hooks:
-    -   id: blacken-docs
--   repo: https://github.com/PyCQA/flake8
+      - id: blacken-docs
+  - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:
-    -   id: flake8
+      - id: flake8
         args: ['--config', 'api/.flake8']
         exclude: (docs/|get_latest_release_changelog.py)
-# -   repo: https://github.com/PyCQA/pylint
-#     rev: v2.15.5
-#     hooks:
-#     -   id: pylint
-#         exclude: (docs/|get_latest_release_changelog.py)
--   repo: https://github.com/antonbabenko/pre-commit-terraform
+  # -   repo: https://github.com/PyCQA/pylint
+  #     rev: v2.15.5
+  #     hooks:
+  #     -   id: pylint
+  #         exclude: (docs/|get_latest_release_changelog.py)
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
     rev: v1.81.0
     hooks:
-    -   id: terraform_fmt
+      - id: terraform_fmt
         exclude: '^(?!infrastructure/).*'
-    -   id: terraform_validate
+      - id: terraform_validate
         exclude: '^(?!infrastructure/).*'
-    -   id: terraform_docs
+      - id: terraform_docs
         args:
-            - markdown table --recursive --output-file README.md .
+          - markdown table --recursive --output-file README.md .
         exclude: '^(?!infrastructure/).*'
--   repo: https://github.com/bridgecrewio/checkov.git
+  - repo: https://github.com/bridgecrewio/checkov.git
     rev: 2.3.261
     hooks:
-    -   id: checkov
+      - id: checkov
         args: [--quiet, --compact]
         exclude: '^(?!infrastructure/).*'
--   repo: local
+  - repo: local
     hooks:
-    -   id: sdk_test
+      - id: sdk_test
         name: sdk_test
         language: system
         entry: bash -c 'make sdk-test'
         files: sdk/*.
         pass_filenames: false
--   repo: local
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
     hooks:
-    -   id: ui_test
+      - id: prettier
+        args: ['--config', 'ui/.prettierrc.json', './ui']
+  - repo: local
+    hooks:
+      - id: ui_test
         name: ui_test
         language: system
         entry: bash -c 'cd ./ui; npm run test:all'

Makefile

@@ -11,9 +11,7 @@ help: 				## List targets and description
 precommit:
 	pre-commit install
 
-security-check:
-	@$(MAKE) detect-secrets
-	@$(MAKE) detect-vulnerabilities
+security-check: detect-secrets detect-vulnerabilities
 
 detect-secrets:
 	@git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline
@@ -66,7 +64,7 @@ api-create-local-venv:		## Create the api local venv for deployment
 	@cd api/; ./local-venv-setup.sh
 
 api-create-image:		## Manually (re)create the api environment image
-	@cd api/; ./batect runtime-environment
+	@cd api/; ./batect --tag-image service-image=rapid-api-service-image runtime-environment
 
 api-shell:			## Run the api application and drop me into a shell
 	@cd api/; ./batect shell
@@ -88,8 +86,8 @@ api-tag-and-upload-release-image:## Tag and upload the api release image
 api-tag-prod-candidate:		## Tag the uploaded api image as a candidate for PROD deployment
 	@cd api/; $(MAKE) tag-prod-candidate
 
-api-tag-live-in-prod:		## Deploy the latest version of the api
-	@cd api/; $(MAKE) tag-live-in-prod
+api-app-live-in-prod:		## Deploy the latest version of the api
+	@cd api/; $(MAKE) app-live-in-prod
 
 api-check-app-is-running:
 	@cd api/; $(MAKE) check-app-is-running
@@ -163,6 +161,12 @@ ui-run-dev:			## Run the ui application with hot reload
 ui-test:			## Test ui site
 	@cd ui/; npm run test:all
 
+ui-test-e2e:
+	@cd ui/; npx playwright test ui/playwright
+
+ui-test-e2e-headed:
+	@cd ui/; npx playwright test ui/playwright --ui
+
 # UI Release --------------------
 ##
 ui-create-static-out:

api/Makefile

@@ -1,5 +1,5 @@
-ECS_SERVICE=rapid-ecs-service
-ECS_CLUSTER=rapid-cluster
+ECS_SERVICE=rapid-preprod-ecs-service
+ECS_CLUSTER=rapid-preprod-cluster
 LATEST_COMMIT_HASH=$(shell git rev-parse --short HEAD)
 ACCOUNT_ECR_URI=$(AWS_ACCOUNT).dkr.ecr.$(AWS_REGION).amazonaws.com
 IMAGE_NAME=data-f1-registry

api/api/controller/auth.py

@@ -65,7 +65,9 @@ async def redirect_oauth_token_request(request: Request):
     }
     payload = await _load_json_bytes_to_dict(request)
 
-    response = requests.post(IDENTITY_PROVIDER_TOKEN_URL, headers=headers, data=payload)
+    response = requests.post(
+        IDENTITY_PROVIDER_TOKEN_URL, headers=headers, data=payload, timeout=5
+    )
 
     return response.json()
 
@@ -114,7 +116,7 @@ async def _get_access_token(auth, code, cognito_user_login_client_id):
         "code": code,
     }
     response = requests.post(
-        IDENTITY_PROVIDER_TOKEN_URL, auth=auth, headers=headers, data=payload
+        IDENTITY_PROVIDER_TOKEN_URL, auth=auth, headers=headers, data=payload, timeout=5
     )
     response_content = json.loads(response.content.decode(CONTENT_ENCODING))
     access_token = response_content["access_token"]

api/api/controller/datasets.py

@@ -43,7 +43,6 @@
 from api.domain.dataset_filters import DatasetFilters
 from api.domain.dataset_metadata import DatasetMetadata
 from api.domain.schema_metadata import SchemaMetadata
-from api.domain.metadata_search import metadata_search_query
 from api.domain.mime_type import MimeType
 from api.domain.sql_query import SQLQuery
 from api.domain.Jobs.Job import generate_uuid
@@ -128,11 +127,7 @@ class EnrichedMetadata(SchemaMetadata):
         include_in_schema=False,
     )
     async def search_dataset_metadata(term: str):
-        sql_query = metadata_search_query(term)
-        df = athena_adapter.query_sql(sql_query)
-        df["version"] = df["version"].fillna(value="0")
-        df["data"] = df["data"].fillna(value="")
-        return df.to_dict("records")
+        return None
 
 
 @datasets_router.get(

api/test/api/controller/test_auth.py

@@ -42,5 +42,6 @@ def test_calls_cognito_for_access_token_when_callback_is_called_with_temporary_c
                 "redirect_uri": COGNITO_REDIRECT_URI,
                 "code": temporary_code,
             },
+            timeout=5,
         )
         mock_redirect.assert_called_once_with(url="/", status_code=HTTP_302_FOUND)