Taskfile.yml

version: '3'

dotenv:
  - api/.env
  - ui/.env
  - email/.env

env:
  COMMIT_SHA:
    sh: git rev-parse --short HEAD

vars:
  PROJECT_NAME: giga-dataops-platform_ingestion-portal

tasks:
  default:
    desc: Build and start Docker containers
    cmds:
      - docker compose --project-name giga-dataops-platform --file docker-compose-network.yaml up --detach --build --remove-orphans {{.CLI_ARGS}}
      - docker compose --project-name {{.PROJECT_NAME}} up --detach --build --remove-orphans {{.CLI_ARGS}}

  setup:
    desc: Run first-time setup tasks
    cmds:
      - task: setup-pre-commit
      - task: setup-python
      - task: setup-node

  setup-helm:
    internal: true
    desc: Add Helm repos
    cmds:
      - helm repo add bitnami https://charts.bitnami.com/bitnami
      - helm repo update

  setup-pre-commit:
    internal: true
    desc: Install pre-commit hooks
    cmds:
      - pip install -U pre-commit
      - pre-commit install --install-hooks -t pre-commit -t commit-msg

  setup-python:
    internal: true
    dir: api
    desc: Install Python dependencies
    cmds:
      - poetry env use 3.11
      - poetry install --no-root

  setup-node:
    internal: true
    dir: ui
    desc: Install Node dependencies
    cmds:
      - npm install -D

  logs:
    desc: Attach to Docker logs
    cmds:
      - docker compose --project-name {{.PROJECT_NAME}} logs --follow {{.CLI_ARGS}}

  stop:
    desc: Stop containers
    cmds:
      - docker compose --project-name {{.PROJECT_NAME}} stop {{.CLI_ARGS}}

  restart:
    desc: Restart containers
    cmds:
      - docker compose --project-name {{.PROJECT_NAME}} restart {{.CLI_ARGS}}

  exec:
    desc: Run a command in the specified container
    cmds:
      - docker compose --project-name {{.PROJECT_NAME}} exec {{.CLI_ARGS}}

  clean:
    desc: Remove containers
    cmds:
      - docker compose --project-name {{.PROJECT_NAME}} down --volumes --remove-orphans {{.CLI_ARGS}}

  test-api:
    desc: Run API unit tests
    dir: api
    cmds:
      - poetry run pytest

  build-and-push:
    desc: Build and push Docker image
    cmds:
      - task: build-cr
      - task: push-cr

  build-cr:
    internal: true
    desc: Build Docker image
    cmds:
      - >
        docker build --tag {{.INGESTION_PORTAL_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}}
        --build-arg "VITE_SENTRY_DSN={{.VITE_SENTRY_DSN}}"
        --build-arg "VITE_AZURE_CLIENT_ID={{.VITE_AZURE_CLIENT_ID}}"
        --build-arg "VITE_AZURE_TENANT_ID={{.VITE_AZURE_TENANT_ID}}"
        --build-arg "VITE_AZURE_TENANT_NAME={{.VITE_AZURE_TENANT_NAME}}"
        --build-arg "VITE_AZURE_SUSI_AUTH_POLICY_NAME={{.VITE_AZURE_SUSI_AUTH_POLICY_NAME}}"
        --build-arg "VITE_AZURE_PASSWORD_RESET_AUTH_POLICY_NAME={{.VITE_AZURE_PASSWORD_RESET_AUTH_POLICY_NAME}}"
        --build-arg "VITE_DATAHUB_URL=https://io-datahub-dev.unitst.org"
        -f Dockerfile.prod .
      - docker build --tag {{.EMAIL_SERVICE_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}} -f ./email/Dockerfile.prod ./email

  push-cr:
    internal: true
    desc: Push Docker image
    cmds:
      - docker push {{.INGESTION_PORTAL_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}}
      - docker push {{.EMAIL_SERVICE_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}}

  k8s-prereqs:
    desc: Install K8s prerequisites
    dir: infra/k8s
    cmds:
      - kubectl apply -f namespace.yaml
      - kubectl apply -f secrets.yaml --namespace giga-data-ingestion
      - kubectl apply -f configmap-prd.yaml --namespace giga-data-ingestion

  helm-up:
    desc: Install/upgrade Helm chart to a K8s cluster running on localhost
    dir: infra
    cmds:
      - task: k8s-prereqs
      - >
        helm upgrade --install postgres bitnami/postgresql
        --version 14.0.5
        --namespace giga-data-ingestion
        --values ./helm/pg-values.yaml
        --values ./helm/pg-values-override-local.yaml
      - >
        helm upgrade --install redis bitnami/redis
        --version 18.18.0
        --namespace giga-data-ingestion
        --values ./helm/redis-values.yaml
        --values ./helm/redis-values-override-local.yaml
      - >
        helm upgrade --install email-service ./helm/email-service
        --namespace giga-data-ingestion
        --set image.repository={{.EMAIL_SERVICE_IMAGE_REPOSITORY}}
        --set image.tag={{.COMMIT_SHA}}
      - >
        helm upgrade --install ingestion-portal ./helm/data-ingestion
        --namespace giga-data-ingestion
        --values ./helm/data-ingestion.values-override-local.yaml
        --set image.repository={{.INGESTION_PORTAL_IMAGE_REPOSITORY}}
        --set image.tag={{.COMMIT_SHA}}

  helm-upi:
    desc: Install/upgrade Helm chart to a K8s cluster with configured HTTPS ingress
    dir: infra
    cmds:
      - task: k8s-prereqs
      - >
        helm upgrade --install postgres bitnami/postgresql
        --version 14.0.5
        --namespace giga-data-ingestion
        --values ./helm/pg-values.yaml
        --values ./helm/pg-values-override-local.yaml
        --set auth.existingSecret=giga-ingestion-portal-secrets
      - >
        helm upgrade --install redis bitnami/redis
        --version 18.18.0
        --namespace giga-data-ingestion
        --values ./helm/redis-values.yaml
        --values ./helm/redis-values-override-local.yaml
        --set auth.existingSecret=giga-ingestion-portal-secrets
      - >
        helm upgrade --install email-service ./helm/email-service
        --namespace giga-data-ingestion
        --values ./helm/email-service.values-override-prd.yaml
        --set image.repository={{.EMAIL_SERVICE_IMAGE_REPOSITORY}}
        --set image.tag={{.COMMIT_SHA}}
        --set envFrom[0].configMapRef.name=giga-ingestion-portal-configmap
        --set envFrom[1].secretRef.name=giga-ingestion-portal-secrets
      - >
        helm upgrade --install ingestion-portal ./helm/data-ingestion
        --namespace giga-data-ingestion
        --values ./helm/data-ingestion.values-override-local-ingress.yaml
        --values ./helm/data-ingestion.values-override-prd.yaml
        --set image.repository={{.INGESTION_PORTAL_IMAGE_REPOSITORY}}
        --set image.tag={{.COMMIT_SHA}}
        --set ingress.hosts[0].host={{.INGESTION_PORTAL_INGRESS_HOST}}
        --set envFrom[0].configMapRef.name=giga-ingestion-portal-configmap
        --set envFrom[1].secretRef.name=giga-ingestion-portal-secrets

  helm-down:
    desc: Uninstall Helm chart
    dir: infra/helm
    ignore_error: true
    cmds:
      - helm uninstall ingestion-portal --namespace giga-data-ingestion
      - helm uninstall email-service --namespace giga-data-ingestion
      - helm uninstall redis --namespace giga-data-ingestion
      - helm uninstall postgres --namespace giga-data-ingestion

  helm-clean:
    desc: Delete K8s resources
    cmds:
      - kubectl delete all --all --namespace giga-data-ingestion
      - kubectl delete pvc --all --namespace giga-data-ingestion

  makemigrations:
    desc: Generate database migrations
    cmds:
      - task exec -- api poetry run alembic revision --autogenerate -m "{{.CLI_ARGS}}"

  generate-migration:
    desc: Generate skeleton database migration file
    cmds:
      - task exec -- api poetry run alembic revision -m "{{.CLI_ARGS}}"

  migrate:
    desc: Run database migrations
    cmds:
      - task exec -- api poetry run alembic upgrade head

  downgrade:
    desc: Downgrade database migrations
    cmds:
      - task exec -- api poetry run alembic downgrade "{{.CLI_ARGS}}"

  psql:
    desc: Open a psql shell
    cmds:
      - task exec -- db psql -U {{.POSTGRESQL_USERNAME}} -d {{.POSTGRESQL_DATABASE}}

  redis-cli:
    desc: Open a redis-cli shell
    cmds:
      - task exec -- -it redis redis-cli --pass {{.REDIS_PASSWORD}}

  generate-qos-fixtures:
    desc: Generate QoS fixtures
    cmds:
      - task exec -- api poetry run python -m scripts.generate_qos_fixtures

  load-fixtures:
    desc: Install database fixtures
    cmds:
      - task exec -- api poetry run python -m scripts.load_fixtures {{.CLI_ARGS}}

  test-build:
    desc: Test UI build
    cmds:
      - rm -rf build
      - >
        docker build --tag {{.INGESTION_PORTAL_IMAGE_REPOSITORY}}:{{.COMMIT_SHA}}
        --build-arg "VITE_SENTRY_DSN={{.VITE_SENTRY_DSN}}"
        --build-arg "VITE_AZURE_CLIENT_ID={{.VITE_AZURE_CLIENT_ID}}"
        --build-arg "VITE_AZURE_TENANT_ID={{.VITE_AZURE_TENANT_ID}}"
        --build-arg "VITE_AZURE_TENANT_NAME={{.VITE_AZURE_TENANT_NAME}}"
        --build-arg "VITE_AZURE_SUSI_AUTH_POLICY_NAME={{.VITE_AZURE_SUSI_AUTH_POLICY_NAME}}"
        --build-arg "VITE_AZURE_PASSWORD_RESET_AUTH_POLICY_NAME={{.VITE_AZURE_PASSWORD_RESET_AUTH_POLICY_NAME}}"
        --build-arg "VITE_DATAHUB_URL=https://io-datahub-dev.unitst.org"
        --build-arg "VITE_REDIRECT_URL=http://localhost:3000"
        --target build
        -f Dockerfile.prod .

  generate-b2c-policy:
    desc: Generate AAD B2C policy
    dir: azure/b2c-policy/LocalLogin
    cmds:
      - sed -e 's|${TENANT_NAME}|{{.AZURE_TENANT_NAME}}|g' template/SignUpOrSignin.xml > dev/SignUpOrSignin.xml
      - sed -e 's|${TENANT_NAME}|{{.AZURE_TENANT_NAME}}|g' template/TrustFrameworkBase.xml > dev/TrustFrameworkBase.xml
      - sed -e 's|${TENANT_NAME}|{{.AZURE_TENANT_NAME}}|g' template/TrustFrameworkExtensions.xml > dev/TrustFrameworkExtensions.xml
      - >
        sed -e 's|${TENANT_NAME}|{{.AZURE_TENANT_NAME}}|g'
        -e 's|${INGRESS_HOST}|io-dataingestion-dev.unitst.org|'
        template/TrustFrameworkExtensions_groups.xml > dev/TrustFrameworkExtensions_groups.xml
      - sed -e 's|${TENANT_NAME}|{{.AZURE_TENANT_NAME}}|g' template/TrustFrameworkLocalization.xml > dev/TrustFrameworkLocalization.xml

      - sed -e 's|${TENANT_NAME}|gigasyncteststg|g' template/SignUpOrSignin.xml > stg/SignUpOrSignin.xml
      - sed -e 's|${TENANT_NAME}|gigasyncteststg|g' template/TrustFrameworkBase.xml > stg/TrustFrameworkBase.xml
      - sed -e 's|${TENANT_NAME}|gigasyncteststg|g' template/TrustFrameworkExtensions.xml > stg/TrustFrameworkExtensions.xml
      - >
        sed -e 's|${TENANT_NAME}|gigasyncteststg|g'
        -e 's|${INGRESS_HOST}|io-dataingestion-stg.unitst.org|'
        template/TrustFrameworkExtensions_groups.xml > stg/TrustFrameworkExtensions_groups.xml
      - sed -e 's|${TENANT_NAME}|gigasyncteststg|g' template/TrustFrameworkLocalization.xml > stg/TrustFrameworkLocalization.xml

      - sed -e 's|${TENANT_NAME}|gigasynctestprd|g' template/SignUpOrSignin.xml > prd/SignUpOrSignin.xml
      - sed -e 's|${TENANT_NAME}|gigasynctestprd|g' template/TrustFrameworkBase.xml > prd/TrustFrameworkBase.xml
      - sed -e 's|${TENANT_NAME}|gigasynctestprd|g' template/TrustFrameworkExtensions.xml > prd/TrustFrameworkExtensions.xml
      - >
        sed -e 's|${TENANT_NAME}|gigasynctestprd|g'
        -e 's|${INGRESS_HOST}|sync.giga.global|'
        template/TrustFrameworkExtensions_groups.xml > prd/TrustFrameworkExtensions_groups.xml
      - sed -e 's|${TENANT_NAME}|gigasynctestprd|g' template/TrustFrameworkLocalization.xml > prd/TrustFrameworkLocalization.xml

  ipython:
    desc: Open an IPython shell
    cmds:
      - task exec -- api poetry run ipython