Split out Airbyte Frontend and Cli build. (#11064)

Split out the build so builds are triggered based on files that are changed. This should help with the overwhelming builds we currently have, reduce the Github api load we see, and help save AWS costs.

Here we tackle the frontend and cli builds first. Follow up PRs will tackle the connector and platform builds.

General build triggering rules:
- always trigger if the files directly related to the build are modified.
- always trigger on master.
- always trigger if .github is modified to test all build changes are compatible.
- for the frontend, we also want to trigger if backend changes happen since some set of these changes can affect frontend.

- Use https://github.com/dorny/paths-filter that lets us filter on paths and returns true/false if that path has been -modified. This was the best among the alternatives of using the default GHA path syntax (too restrictive), or using the next best alternative library(even uglier/complex if conditionals) I could find with an hour of googling.
- Add the relevant conditionals in the frontend and octavia-cli builds.
- For the frontend build, it is sufficient to add this conditional in the start step, since the rest of the steps require the start step to run for themselves to succeed.

.github/workflows/gradle.yml

@@ -8,7 +8,54 @@ on:
       - "gitbook/v1"
 
 jobs:
-  ## Gradle Build (Connectors Base)
+  # COMMON TASKS
+  ensure-images-exist:
+    name: "Ensure all required Docker images exist on Dockerhub"
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - name: Check images exist
+        run: ./tools/bin/check_images_exist.sh all
+  # The output of this job is used to trigger the following builds.
+  changes:
+    name: "Detect Modified Files"
+    runs-on: ubuntu-latest
+    outputs:
+      backend: ${{ steps.filter.outputs.backend }}
+      build: ${{ steps.filter.outputs.build }}
+      cli: ${{ steps.filter.outputs.cli }}
+      frontend: ${{ steps.filter.outputs.frontend }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            backend:
+              - 'airbyte-**/**'
+            build:
+              - '.github/**'
+            cli:
+              - 'airbyte-cli/**'
+            frontend:
+              - 'airbyte-webapp/**'
+              - 'airbyte-webapp-e2e-tests/**'
+
+  # Uncomment to debug.
+  #  changes-output:
+  #    name: "Debug Change Detection Logic"
+  #    needs: changes
+  #    runs-on: ubuntu-latest
+  #    steps:
+  #      - uses: actions/checkout@v2
+  #      - run: |
+  #          echo '${{ toJSON(needs) }}'
+
+  # Gradle Build (Connectors Base)
   # In case of self-hosted EC2 errors, remove this block.
   start-connectors-base-build-runner:
     name: "Connectors Base: Start Build EC2 Runner"
@@ -28,17 +75,6 @@ jobs:
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
 
-  ensure-images-exist:
-    name: "Ensure all required Docker images exist on Dockerhub"
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v2
-
-      - name: Check images exist
-        run: ./tools/bin/check_images_exist.sh all
-
   build-connectors-base:
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
     needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
@@ -137,7 +173,7 @@ jobs:
           label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}
 
-  ## Gradle Build (Platform)
+  # Gradle Build (Platform)
   # In case of self-hosted EC2 errors, remove this block.
   start-platform-build-runner:
     name: "Platform: Start Build EC2 Runner"
@@ -267,6 +303,7 @@ jobs:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+
   platform-new-scheduler-acceptance:
     # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
     needs: start-platform-new-scheduler-acceptance-runner # required to start the main job when the runner is ready
@@ -393,7 +430,9 @@ jobs:
           ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}
 
   octavia-cli-build:
+    needs: changes
     runs-on: ubuntu-latest
+    if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/master'
     name: "Octavia CLI: Build"
     timeout-minutes: 90
     steps:
@@ -464,6 +503,8 @@ jobs:
   # In case of self-hosted EC2 errors, remove this block.
   start-frontend-test-runner:
     name: Start Frontend Test EC2 Runner
+    needs: changes
+    if: needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || needs.changes.outputs.backend == 'true' || github.ref == 'refs/heads/master'
     timeout-minutes: 10
     runs-on: ubuntu-latest
     outputs:
@@ -539,7 +580,9 @@ jobs:
       - start-frontend-test-runner # required to get output from the start-runner job
       - frontend-test # required to wait when the main job is done
     runs-on: ubuntu-latest
-    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+    # Thus, we check for skipped here.
+    if: ${{ always() && needs.start-frontend-test-runner.result != 'skipped'}}
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1