feat(ci): Worker deploy queues matrix (#4975)

* feat(ci): deploy queues

* fix: pipeline

* fix: from json

* fix: to string

* fix: test

* fix: a

* fix: worker

* fix: service matrix input

* fix: general queue

* fix: dev env

.github/workflows/dev-deploy-worker.yml

@@ -31,7 +31,7 @@ jobs:
       ee: ${{ contains (matrix.name,'-ee') }}
     secrets: inherit
 
-  deploy_dev_worker:
+  build_dev_worker:
     if: "!contains(github.event.head_commit.message, 'ci skip')"
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
@@ -61,96 +61,32 @@ jobs:
           docker_name: ${{ matrix.name }}
           bullmq_secret: ${{ secrets.BULL_MQ_PRO_NPM_TOKEN }}
 
-      - name: Checkout cloud infra
-        if: ${{ contains (matrix.name,'-ee') }}
-        uses: actions/checkout@master
-        with:
-          repository: novuhq/cloud-infra
-          token: ${{ secrets.GH_PACKAGES }}
-          path: cloud-infra
-
-      - name: Configure AWS credentials
-        if: ${{ contains (matrix.name,'-ee') }}
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
-
-      - name: Terraform setup
-        uses: hashicorp/setup-terraform@v1
-        if: ${{ contains (matrix.name,'-ee') }}
-        with:
-          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
-          terraform_version: 1.5.5
-          terraform_wrapper: false
-
-      - name: Terraform Init
-        if: ${{ contains (matrix.name,'-ee') }}
-        working-directory: cloud-infra/terraform/novu/aws
-        run: terraform init
-
-      - name: Terraform get output
-        working-directory: cloud-infra/terraform/novu/aws
-        if: ${{ contains (matrix.name,'-ee') }}
-        id: terraform
-        run: |
-          echo "worker_ecs_container_name=$(terraform output -json worker_ecs_container_name | jq -r .)" >> $GITHUB_ENV
-          echo "worker_ecs_service=$(terraform output -json worker_ecs_service | jq -r .)" >> $GITHUB_ENV
-          echo "worker_ecs_cluster=$(terraform output -json worker_ecs_cluster | jq -r .)" >> $GITHUB_ENV
-          echo "worker_task_name=$(terraform output -json worker_task_name | jq -r .)" >> $GITHUB_ENV
-
-      - name: Download task definition
-        if: ${{ contains (matrix.name,'-ee') }}
-        run: |
-          aws ecs describe-task-definition --task-definition ${{ env.worker_task_name }} \
-          --query taskDefinition > task-definition.json
-
-      - name: Render Amazon ECS task definition
-        if: ${{ contains (matrix.name,'-ee') }}
-        id: render-web-container
-        uses: aws-actions/amazon-ecs-render-task-definition@v1
-        with:
-          task-definition: task-definition.json
-          container-name: ${{ env.worker_ecs_container_name }}
-          image: ${{ steps.docker_build.outputs.image }}
-
-      - name: Deploy to Amazon ECS service
-        if: ${{ contains (matrix.name,'-ee') }}
-        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
-        with:
-          task-definition: ${{ steps.render-web-container.outputs.task-definition }}
-          service: ${{ env.worker_ecs_service }}
-          cluster: ${{ env.worker_ecs_cluster }}
-          wait-for-service-stability: true
-
-      - name: get-npm-version
-        id: package-version
-        if: ${{ contains (matrix.name,'-ee') }}
-        uses: martinbeentjes/npm-get-version-action@main
-        with:
-          path: apps/worker
+  # Temporary for the migration phase
+  deploy_general_worker:
+    needs: build_dev_worker
+    uses: ./.github/workflows/reusable-app-service-deploy.yml
+    secrets: inherit
+    with:
+      environment: Development
+      service_name: worker
+      terraform_workspace: novu-dev
+      # This is a workaround to an issue with matrix outputs
+      docker_image: ghcr.io/novuhq/novu/worker-ee:${{ github.sha }}
 
-      - name: Create Sentry release
-        if: ${{ contains (matrix.name,'-ee') }}
-        uses: getsentry/action-release@v1
-        env:
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          SENTRY_ORG: novu-r9
-          SENTRY_PROJECT: worker
-        with:
-          version: ${{ steps.package-version.outputs.current-version}}
-          environment: dev
-          version_prefix: v
-          sourcemaps: apps/worker/dist
-          ignore_empty: true
-          ignore_missing: true
-          url_prefix: "~"
+  deploy_dev_workers:
+    needs: deploy_general_worker
+    uses: ./.github/workflows/reusable-workers-service-deploy.yml
+    secrets: inherit
+    with:
+      environment: Development
+      terraform_workspace: novu-dev
+      # This is a workaround to an issue with matrix outputs
+      docker_image: ghcr.io/novuhq/novu/worker-ee:${{ github.sha }}
 
   newrelic:
     runs-on: ubuntu-latest
     name: New Relic Deploy
-    needs: deploy_dev_worker
+    needs: deploy_dev_workers
     environment: Development
     steps:
       # This step builds a var with the release tag value to use later

.github/workflows/prod-deploy-worker.yml

@@ -76,6 +76,7 @@ jobs:
           docker push ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$IMAGE_TAG
           echo "IMAGE=ghcr.io/$REGISTRY_OWNER/$DOCKER_NAME:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
+
   deploy_prod_worker_eu:
     needs: build_prod_image
     uses: ./.github/workflows/reusable-app-service-deploy.yml

.github/workflows/reusable-workers-service-deploy.yml

@@ -0,0 +1,103 @@
+name: Deploy Workers Job
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      terraform_workspace:
+        required: true
+        type: string
+      docker_image:
+        required: true
+        type: string
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  infrastructure_data:
+    runs-on: ubuntu-latest
+    timeout-minutes: 80
+    environment: ${{ inputs.environment }}
+    env:
+      TF_WORKSPACE: ${{ inputs.terraform_workspace }}
+    permissions:
+      contents: read
+      deployments: write
+    outputs:
+      services_to_deploy: ${{ steps.terraform.outputs.queue_workers_services }}
+      ecs_cluster: ${{ steps.terraform.outputs.ecs_cluster }}
+      aws_region: ${{ steps.terraform.outputs.aws_region }}
+    steps:
+      - run: echo "Deploying ${{ inputs.service_name }} to ${{ inputs.terraform_workspace }} And Docker Tag ${{ inputs.docker_image }}"
+      - name: Checkout cloud infra
+        uses: actions/checkout@master
+        with:
+          repository: novuhq/cloud-infra
+          token: ${{ secrets.GH_PACKAGES }}
+          path: cloud-infra
+
+      - name: Terraform setup
+        uses: hashicorp/setup-terraform@v1
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
+          terraform_version: 1.5.5
+          terraform_wrapper: false
+
+      - name: Terraform Init
+        working-directory: cloud-infra/terraform/novu/aws
+        run: terraform init
+
+      - name: Terraform get output
+        working-directory: cloud-infra/terraform/novu/aws
+        id: terraform
+        run: |
+          echo "queue_workers_services=$(terraform output -json queue_workers_services)" >> $GITHUB_OUTPUT
+          echo "ecs_cluster=$(terraform output -json worker_ecs_cluster | jq -r .)" >> $GITHUB_OUTPUT
+          echo "aws_region=$(terraform output -json aws_region | jq -r .)" >> $GITHUB_OUTPUT
+
+
+  deploy_worker_queue:
+    needs: infrastructure_data
+    runs-on: ubuntu-latest
+    timeout-minutes: 80
+    environment: ${{ inputs.environment }}
+    env:
+      TF_WORKSPACE: ${{ inputs.terraform_workspace }}
+    permissions:
+      contents: read
+      deployments: write
+    strategy:
+      matrix:
+        worker: ${{fromJson(needs.infrastructure_data.outputs.services_to_deploy)}}
+    steps:
+      - run: echo "Deploying ${{ matrix.name }} to ${{ inputs.terraform_workspace }} And Docker Tag ${{ inputs.docker_image }}"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ needs.infrastructure_data.outputs.aws_region }}
+
+      - name: Download task definition
+        run: |
+          aws ecs describe-task-definition --task-definition ${{ matrix.worker.task_name }} \
+          --query taskDefinition > task-definition.json
+
+      - name: Render Amazon ECS task definition
+        id: render-web-container
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: ${{ matrix.worker.container_name }}
+          image: ${{ inputs.docker_image }}
+
+      - name: Deploy to Amazon ECS service
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        with:
+          task-definition: ${{ steps.render-web-container.outputs.task-definition }}
+          service: ${{ matrix.worker.service }}
+          cluster: ${{ needs.infrastructure_data.outputs.ecs_cluster }}
+          wait-for-service-stability: true