Skip to content

Commit

Permalink
Merge pull request #21 from no10ds/fix/actions
Browse files Browse the repository at this point in the history
Fix/actions
  • Loading branch information
TobyDrane authored Sep 1, 2023
2 parents 5a36759 + 65c39e8 commit 63ea248
Show file tree
Hide file tree
Showing 66 changed files with 1,166 additions and 702 deletions.
13 changes: 7 additions & 6 deletions .env.example
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
AWS_ACCOUNT=
AWS_REGION=
AWS_ACCOUNT=123456
AWS_REGION=eu-west-2

# API Specific
COGNITO_USER_POOL_ID=rapid-pool
DATA_BUCKET=rapid-bucket
DOMAIN_NAME=rapid-domain
RESOURCE_PREFIX=rapid
ALLOWED_EMAIL_DOMAINS=example1.com,example2.com
DATA_BUCKET=the-bucket
RESOURCE_PREFIX=rapid
DOMAIN_NAME=example.com
COGNITO_USER_POOL_ID=11111111
LAYERS=raw,layer

# SDK Specific
RAPID_CLIENT_ID=
Expand Down
2 changes: 2 additions & 0 deletions .github/.github.env
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,5 @@ COGNITO_USER_POOL_ID=rapid-pool
RESOURCE_PREFIX=rapid
ALLOWED_EMAIL_DOMAINS=example1.com,example2.com
LAYERS=raw,layer
DOMAIN_NAME=example.com
DATA_BUCKET=the-bucket
4 changes: 1 addition & 3 deletions .github/workflows/dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ jobs:
- name: Populate .env with additional vars
run: |
cp ./.github/.github.env .env
echo DOMAIN_NAME=${{ secrets.DOMAIN_NAME }} >> .env
echo DATA_BUCKET=${{ secrets.DATA_BUCKET }} >> .env
echo AWS_ACCOUNT=${{ secrets.AWS_ACCOUNT }} >> .env
echo AWS_REGION=${{ secrets.AWS_REGION }} >> .env
echo AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} >> .env
Expand Down Expand Up @@ -76,7 +74,7 @@ jobs:
run: |
echo "TWINE_USERNAME=${{ secrets.TWINE_USERNAME_TEST }}" >> .env
echo "TWINE_PASSWORD=${{ secrets.TWINE_PASSWORD_TEST }}" >> .env
echo TWINE_NON_INTERACTIVE=${{ secrets.TWINE_NON_INTERACTIVE }} >> .env
echo "TWINE_NON_INTERACTIVE=true" >> .env
- name: Setup Python
uses: actions/setup-python@v4
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ jobs:
- name: Populate .env with additional vars
run: |
cp ./.github/.github.env .env
echo DOMAIN_NAME=${{ secrets.DOMAIN_NAME }} >> .env
echo DATA_BUCKET=${{ secrets.DATA_BUCKET }} >> .env
echo AWS_ACCOUNT=${{ secrets.AWS_ACCOUNT }} >> .env
echo AWS_REGION=${{ secrets.AWS_REGION }} >> .env
echo AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} >> .env
Expand All @@ -69,7 +67,7 @@ jobs:
run: make api-tag-prod-candidate

- name: API Deploy Image to Prod
run: make api-tag-live-in-prod
run: make api-app-live-in-prod

- name: API Allow for Application to Start
run: sleep 120
Expand All @@ -81,6 +79,8 @@ jobs:
- name: API E2E Tests
id: e2e-tests
env:
DOMAIN_NAME: ${{ secrets.DOMAIN_NAME }}
DATA_BUCKET: ${{ secrets.DATA_BUCKET }}
COGNITO_USER_POOL_ID: ${{ secrets.COGNITO_USER_POOL_ID }}
RESOURCE_PREFIX: ${{ secrets.RESOURCE_PREFIX }}
ALLOWED_EMAIL_DOMAINS: ${{ secrets.ALLOWED_EMAIL_DOMAINS }}
Expand Down
15 changes: 6 additions & 9 deletions .github/workflows/ui-tests.yml
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
name: rAPId Integration Tests

on:
push:
branches:
- "**"

workflow_dispatch:

pull_request:
types:
- opened

jobs:

run-ui-test:
runs-on: self-hosted

Expand All @@ -29,11 +24,13 @@ jobs:
npm install
- name: Install playwright browsers
run: npx playwright install-deps && npx playwright install
run: |
cd ui
npm install @playwright/test -D
- name: run playwright tests
run: npx playwright test ui/playwright
- name: Run playwright tests
run: make ui-test-e2e
env:
DOMAIN: ${{ secrets.DOMAIN }}
DOMAIN_NAME: "https://${{ secrets.DOMAIN_NAME }}"
RESOURCE_PREFIX: ${{ secrets.RESOURCE_PREFIX }}
AWS_REGION: ${{ secrets.AWS_REGION }}
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -182,5 +182,6 @@ docs/_build/
.terraform/
.terraform.lock.hcl

playwright/.auth
playwright/.downloads
ui/playwright/.auth
ui/playwright/.downloads
ui/test-results/
69 changes: 37 additions & 32 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,70 +1,75 @@
exclude: '^ui'
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: check-json
- id: check-merge-conflict
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/PyCQA/bandit
- id: check-yaml
- id: check-json
- id: check-merge-conflict
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/PyCQA/bandit
rev: 1.7.5
hooks:
- id: bandit
- id: bandit
exclude: '(test|docs)/*'
- repo: https://github.com/psf/black
- repo: https://github.com/psf/black
rev: 22.6.0
hooks:
- id: black
- repo: https://github.com/Yelp/detect-secrets
- id: black
- repo: https://github.com/Yelp/detect-secrets
rev: v1.3.0
hooks:
- id: detect-secrets
- id: detect-secrets
exclude: docs/
- repo: https://github.com/asottile/blacken-docs
- repo: https://github.com/asottile/blacken-docs
rev: v1.12.1
hooks:
- id: blacken-docs
- repo: https://github.com/PyCQA/flake8
- id: blacken-docs
- repo: https://github.com/PyCQA/flake8
rev: 4.0.1
hooks:
- id: flake8
- id: flake8
args: ['--config', 'api/.flake8']
exclude: (docs/|get_latest_release_changelog.py)
# - repo: https://github.com/PyCQA/pylint
# rev: v2.15.5
# hooks:
# - id: pylint
# exclude: (docs/|get_latest_release_changelog.py)
- repo: https://github.com/antonbabenko/pre-commit-terraform
# - repo: https://github.com/PyCQA/pylint
# rev: v2.15.5
# hooks:
# - id: pylint
# exclude: (docs/|get_latest_release_changelog.py)
- repo: https://github.com/antonbabenko/pre-commit-terraform
rev: v1.81.0
hooks:
- id: terraform_fmt
- id: terraform_fmt
exclude: '^(?!infrastructure/).*'
- id: terraform_validate
- id: terraform_validate
exclude: '^(?!infrastructure/).*'
- id: terraform_docs
- id: terraform_docs
args:
- markdown table --recursive --output-file README.md .
- markdown table --recursive --output-file README.md .
exclude: '^(?!infrastructure/).*'
- repo: https://github.com/bridgecrewio/checkov.git
- repo: https://github.com/bridgecrewio/checkov.git
rev: 2.3.261
hooks:
- id: checkov
- id: checkov
args: [--quiet, --compact]
exclude: '^(?!infrastructure/).*'
- repo: local
- repo: local
hooks:
- id: sdk_test
- id: sdk_test
name: sdk_test
language: system
entry: bash -c 'make sdk-test'
files: sdk/*.
pass_filenames: false
- repo: local
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v3.0.3
hooks:
- id: ui_test
- id: prettier
args: ['--config', 'ui/.prettierrc.json', './ui']
- repo: local
hooks:
- id: ui_test
name: ui_test
language: system
entry: bash -c 'cd ./ui; npm run test:all'
Expand Down
16 changes: 10 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ help: ## List targets and description
precommit:
pre-commit install

security-check:
@$(MAKE) detect-secrets
@$(MAKE) detect-vulnerabilities
security-check: detect-secrets detect-vulnerabilities

detect-secrets:
@git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline
Expand Down Expand Up @@ -66,7 +64,7 @@ api-create-local-venv: ## Create the api local venv for deployment
@cd api/; ./local-venv-setup.sh

api-create-image: ## Manually (re)create the api environment image
@cd api/; ./batect runtime-environment
@cd api/; ./batect --tag-image service-image=rapid-api-service-image runtime-environment

api-shell: ## Run the api application and drop me into a shell
@cd api/; ./batect shell
Expand All @@ -88,8 +86,8 @@ api-tag-and-upload-release-image:## Tag and upload the api release image
api-tag-prod-candidate: ## Tag the uploaded api image as a candidate for PROD deployment
@cd api/; $(MAKE) tag-prod-candidate

api-tag-live-in-prod: ## Deploy the latest version of the api
@cd api/; $(MAKE) tag-live-in-prod
api-app-live-in-prod: ## Deploy the latest version of the api
@cd api/; $(MAKE) app-live-in-prod

api-check-app-is-running:
@cd api/; $(MAKE) check-app-is-running
Expand Down Expand Up @@ -163,6 +161,12 @@ ui-run-dev: ## Run the ui application with hot reload
ui-test: ## Test ui site
@cd ui/; npm run test:all

ui-test-e2e:
@cd ui/; npx playwright test ui/playwright

ui-test-e2e-headed:
@cd ui/; npx playwright test ui/playwright --ui

# UI Release --------------------
##
ui-create-static-out:
Expand Down
4 changes: 2 additions & 2 deletions api/Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
ECS_SERVICE=rapid-ecs-service
ECS_CLUSTER=rapid-cluster
ECS_SERVICE=rapid-preprod-ecs-service
ECS_CLUSTER=rapid-preprod-cluster
LATEST_COMMIT_HASH=$(shell git rev-parse --short HEAD)
ACCOUNT_ECR_URI=$(AWS_ACCOUNT).dkr.ecr.$(AWS_REGION).amazonaws.com
IMAGE_NAME=data-f1-registry
Expand Down
6 changes: 4 additions & 2 deletions api/api/controller/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,9 @@ async def redirect_oauth_token_request(request: Request):
}
payload = await _load_json_bytes_to_dict(request)

response = requests.post(IDENTITY_PROVIDER_TOKEN_URL, headers=headers, data=payload)
response = requests.post(
IDENTITY_PROVIDER_TOKEN_URL, headers=headers, data=payload, timeout=5
)

return response.json()

Expand Down Expand Up @@ -114,7 +116,7 @@ async def _get_access_token(auth, code, cognito_user_login_client_id):
"code": code,
}
response = requests.post(
IDENTITY_PROVIDER_TOKEN_URL, auth=auth, headers=headers, data=payload
IDENTITY_PROVIDER_TOKEN_URL, auth=auth, headers=headers, data=payload, timeout=5
)
response_content = json.loads(response.content.decode(CONTENT_ENCODING))
access_token = response_content["access_token"]
Expand Down
7 changes: 1 addition & 6 deletions api/api/controller/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
from api.domain.dataset_filters import DatasetFilters
from api.domain.dataset_metadata import DatasetMetadata
from api.domain.schema_metadata import SchemaMetadata
from api.domain.metadata_search import metadata_search_query
from api.domain.mime_type import MimeType
from api.domain.sql_query import SQLQuery
from api.domain.Jobs.Job import generate_uuid
Expand Down Expand Up @@ -128,11 +127,7 @@ class EnrichedMetadata(SchemaMetadata):
include_in_schema=False,
)
async def search_dataset_metadata(term: str):
sql_query = metadata_search_query(term)
df = athena_adapter.query_sql(sql_query)
df["version"] = df["version"].fillna(value="0")
df["data"] = df["data"].fillna(value="")
return df.to_dict("records")
return None


@datasets_router.get(
Expand Down
59 changes: 0 additions & 59 deletions api/api/domain/metadata_search.py

This file was deleted.

1 change: 1 addition & 0 deletions api/test/api/controller/test_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,6 @@ def test_calls_cognito_for_access_token_when_callback_is_called_with_temporary_c
"redirect_uri": COGNITO_REDIRECT_URI,
"code": temporary_code,
},
timeout=5,
)
mock_redirect.assert_called_once_with(url="/", status_code=HTTP_302_FOUND)
Loading

0 comments on commit 63ea248

Please sign in to comment.