feat: add service state restart workflow

.github/workflows/deploy-service-restart.yml

@@ -0,0 +1,70 @@
+name: Restart Services to refresh memory
+
+run-name: Refresh Service Unit in ${{ inputs.environment }} by @${{ github.actor }}
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        type: string
+        default: "tools"
+
+  workflow_dispatch:
+    inputs:
+      environment:
+        type: choice
+        required: true
+        description: "Where to deploy"
+        default: "tools"
+        options:
+          - tools
+          - production
+
+# We only one one deploy happening at a time per environment, so if one is
+# in progress, we'll wait for it to finish before starting the
+# next one.
+# Note that this will also wait for E2E tests to finish, as per the crons workflows
+concurrency:
+  group: deploy-${{ inputs.environment }}
+  cancel-in-progress: false
+
+jobs:
+
+  restart-rebaser:
+    uses: ./.github/workflows/service-restart.yml
+    with:
+      environment: ${{ inputs.environment }}
+      service: rebaser
+    secrets: inherit
+
+  restart-pinga:
+    needs: restart-rebaser
+    uses: ./.github/workflows/service-restart.yml
+    with:
+      environment: ${{ inputs.environment }}
+      service: pinga
+    secrets: inherit
+
+  restart-sdf:
+    needs: restart-pinga
+    uses: ./.github/workflows/service-restart.yml
+    with:
+      environment: ${{ inputs.environment }}
+      service: sdf
+    secrets: inherit
+
+  e2e-validation:
+    needs:
+      - restart-sdf
+    uses: ./.github/workflows/e2e-validation.yml
+    with:
+      environment: ${{ inputs.environment }}
+    secrets: inherit
+
+  api-test:
+    needs:
+      - restart-sdf
+    uses: ./.github/workflows/run-api-test.yml
+    with:
+      environment: ${{ inputs.environment }}
+    secrets: inherit

.github/workflows/instance-refresh.yml

@@ -4,8 +4,8 @@ run-name: Replacing instances for ${{ inputs.service }}
 
 # Required for IDP JWT and actions/checkout
 permissions:
-      id-token: write
-      contents: read
+  id-token: write
+  contents: read
 
 on:
   workflow_call:

.github/workflows/service-restart.yml

@@ -0,0 +1,52 @@
+name: Service Restart
+
+run-name: Restarts the service on a given node
+
+# Required for IDP JWT and actions/checkout
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        type: string
+        required: true
+        description: "where to deploy"
+      service:
+        type: string
+        required: true
+        description: "service to deploy"
+
+jobs:
+  restart:
+    environment: ${{ inputs.environment }}
+    concurrency:
+      group: instance-restart-${{ inputs.environment }}-${{ inputs.service }}
+      cancel-in-progress: true
+    name: Service Restart
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ vars.AWS_ASSUME_ROLE_ARN }}
+          role-session-name: GitHub_to_AWS_via_FederatedOIDC
+          aws-region: us-east-1
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Toggle maintenance if SDF
+        if:  ${{ inputs.service == 'sdf' }}
+        run: |
+          component/toolbox/awsi.sh toggle-maintenance -p pull-from-env -r us-east-1 -s ${{ inputs.service }} -m y -a y
+      # Without a farily involved re-work this is the easiest way to use our existing tooling to achieve the goal.
+      # Basically just quickly drop them all and bring them all back. Not ideal, but hopefully outside of SDF 
+      # The user impact is extremely minimal/short.
+      - name: Down Service Replicas
+        run: |
+          component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -s ${{ inputs.service }} -S down -a y
+
+      - name: Up Service Replicas
+        run: |
+          component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -s ${{ inputs.service }} -S up -a y