Develop => master for AWS-v87

.gitattributes

@@ -3,4 +3,3 @@
 *.MD text
 *.java text
 *.html text
-docs/api/RESTAPI.md linguist-generated=true

.github/workflows/azure_e2e_run_workflow.yml

@@ -0,0 +1,117 @@
+name: 'Azure e2e - Run Workflow'
+on:
+  schedule:
+    - cron: '0 16 * * *' # UTC 4pm, EST 11am, EDT 12pm
+  workflow_dispatch:
+
+env:
+  BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo
+  RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
+jobs:
+
+  # This job provisions useful parameters for e2e tests
+  params-gen:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    outputs:
+      project-name: ${{ steps.gen.outputs.project_name }}
+      bee-name: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev'
+    steps:
+      - name: Generate a random billing project name
+        id: 'gen'
+        run: |
+          project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30)
+          echo "project_name=${project_name}" >> $GITHUB_OUTPUT
+
+  create-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [params-gen]
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-create
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state
+          # NOTE: For testing/development purposes I'm using dev
+          inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}", "version-template": "dev", "bee-template-name": "rawls-e2e-azure-tests"}'
+
+  create-and-attach-billing-project-to-landing-zone-workflow:
+    runs-on: ubuntu-latest
+    needs: [create-bee-workflow, params-gen]
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: attach-billing-project-to-landing-zone.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{
+            "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ needs.params-gen.outputs.bee-name }}",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "service-account": "firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com" }'
+
+  run-cromwell-az-e2e:
+    needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow]
+    permissions:
+      contents: read
+      id-token: write
+    uses: "broadinstitute/dsp-reusable-workflows/.github/workflows/cromwell-az-e2e-test.yaml@main"
+    with:
+      bee-name: "${{ needs.params-gen.outputs.bee-name }}"
+      billing-project-name: "${{ needs.params-gen.outputs.project-name }}"
+
+  delete-billing-project-v2-from-bee-workflow:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen]
+    if: always()
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ 
+            "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ needs.params-gen.outputs.bee-name }}", 
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}", 
+            "service-account": "firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com", 
+            "silent-on-failure": "false" }'
+
+  destroy-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [params-gen, create-bee-workflow, delete-billing-project-v2-from-bee-workflow]
+    if: always()
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-destroy.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}" }'
+          wait-for-completion: true
+
+  report-workflow:
+    uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main
+    with:
+      notify-slack-channels-upon-workflow-failure: "#cromwell_jenkins_ci_errors"
+    permissions:
+      id-token: write

.github/workflows/chart_update_on_merge.yml

@@ -9,8 +9,14 @@ jobs:
   chart-update:
     name: Cromwhelm Chart Auto Updater
     if: github.event.pull_request.merged == true
-    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    runs-on: ubuntu-latest
     steps:
+    - name: Fetch Jira ID from the commit message
+      id: fetch-jira-id
+      run: |
+        JIRA_ID=$(echo '${{ github.event.pull_request.title }}' | grep -Eo '[A-Z][A-Z]+-[0-9]+' | xargs echo -n | tr '[:space:]' ',')
+        [[ -z "$JIRA_ID" ]] && { echo "No Jira ID found in $1" ; exit 1; }
+        echo "JIRA_ID=$JIRA_ID" >> $GITHUB_OUTPUT
     - name: Clone Cromwell
       uses: actions/checkout@v2
       with:
@@ -49,11 +55,12 @@ jobs:
         username: dsdejenkins
         password: ${{ secrets.DSDEJENKINS_PASSWORD }}
     # Build & push `cromwell`, `womtool`, `cromiam`, and `cromwell-drs-localizer`
+    # This step is validated in the GHA 'docker_build_test.yml' without the accompanying docker push
     - name: Build Cromwell Docker
       run: |
         set -e
         cd cromwell
-        sbt -Dproject.isSnapshot=false -Dproject.isRelease=false dockerBuildAndPush
+        sbt -Dproject.isSnapshot=false dockerBuildAndPush
     - name: Deploy to dev and board release train (Cromwell)
       uses: broadinstitute/repository-dispatch@master
       with:
@@ -68,18 +75,61 @@ jobs:
         repository: broadinstitute/terra-helmfile
         event-type: update-service
         client-payload: '{"service": "cromiam", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}'
-    - name: Edit & push chart
+    - name: Edit & push cromwhelm chart
       env:
         BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
       run: |
         set -e
         cd cromwhelm
         git checkout main
         ls -la
-        sed -i "s/appVersion.*/appVersion: \"$CROMWELL_VERSION\"/" cromwell-helm/Chart.yaml
-        sed -i "s/image: broadinstitute\/cromwell.*/image: broadinstitute\/cromwell:$CROMWELL_VERSION/" cromwell-helm/templates/cromwell.yaml
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" terra-batch-libchart/values.yaml
+
         git diff
         git config --global user.name "broadbot"
         git config --global user.email "broadbot@broadinstitute.org"
-        git commit -am "Auto update to Cromwell $CROMWELL_VERSION"
+        git commit -am "${{ steps.fetch-jira-id.outputs.JIRA_ID }}: Auto update to Cromwell $CROMWELL_VERSION"
         git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/cromwhelm.git main
+        cd -
+
+    - name: Clone terra-helmfile
+      uses: actions/checkout@v3
+      with:
+        repository: broadinstitute/terra-helmfile
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
+        path: terra-helmfile
+
+    - name: Update workflows-app in terra-helmfile
+      run: |
+        set -e
+        cd terra-helmfile
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/workflows-app/values.yaml
+        cd -
+
+    - name: Update cromwell-runner-app in terra-helmfile
+      run: |
+        set -e
+        cd terra-helmfile
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/cromwell-runner-app/values.yaml
+        cd -
+
+
+    - name: Make PR in terra-helmfile
+      env:
+        BROADBOT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+        GH_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+      run: |
+        set -e
+        JIRA_ID=${{ steps.fetch-jira-id.outputs.JIRA_ID }}
+        if [[ $JIRA_ID == "missing" ]]; then
+          echo "JIRA_ID missing, PR to terra-helmfile will not be created"
+          exit 0;
+        fi
+        cd terra-helmfile
+        git checkout -b ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION
+        git config --global user.name "broadbot"
+        git config --global user.email "broadbot@broadinstitute.org"
+        git commit -am "${JIRA_ID}: Auto update Cromwell to $CROMWELL_VERSION in workflows-app and cromwell-runner-app"
+        git push -u origin ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION
+        gh pr create --title "${JIRA_ID}: auto update Cromwell version to $CROMWELL_VERSION in workflows-app and cromwell-runner-app" --body "${JIRA_ID} helm chart update" --label "automerge"
+        cd -