Run End to End test suites as different jobs and fix timeouts (#3241)

Task/Issue URL: https://app.asana.com/0/1203301625297703/1208041207600723/f

**Description**:
- Follow sync end-to-end tests and run test suites type as their own job.
- Address tests timeout that happened randomly in password-autofill by removing some delays
- Fix a test that failed cause the the web page copy changed.

.github/workflows/end-to-end.yml

@@ -6,10 +6,11 @@ on:
   workflow_dispatch:
 
 jobs:
-  end-to-end-tests:
-    name: End to end Tests
+  build-end-to-end-tests:
+    name: Build End to End Tests
     runs-on: macos-14-xlarge
-
+    timeout-minutes: 30
+
     steps:
     - name: Check out the code
       uses: actions/checkout@v3
@@ -48,26 +49,59 @@ jobs:
           ONLY_ACTIVE_ARCH=NO \
         | tee xcodebuild.log
 
-    - name: Install Maestro
-      run: |
-        export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
+    - name: Store Binary
+      uses: actions/upload-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
+
+    - name: Upload logs when workflow failed
+      uses: actions/upload-artifact@v4
+      if: failure() || cancelled()
+      with:
+        name: BuildLogs
+        path: |
+          xcodebuild.log
+          DerivedData/Logs/Test/*.xcresult
+        retention-days: 7
 
-    - name: Release tests
-      run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=release DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+  end-to-end-tests:
+    name: End to end Tests
+    needs: build-end-to-end-tests
+    runs-on: macos-14-xlarge
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        test-tag: [release, privacy, securityTest, adClick]
+      max-parallel: 1 # Uncomment this line to run tests sequentially. 
+      fail-fast: false
 
-    - name: Privacy tests
-      run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=privacy DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+    steps:
+
+    - name: Check out the code
+      uses: actions/checkout@v3 # Don't need submodules here as this is only for the tests folder
+
+    - name: Retrieve Binary
+      uses: actions/download-artifact@v4
+      with:
+        name: duckduckgo-ios-app
+        path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app
 
-    - name: Security tests
+    - name: Install Maestro
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=securityTest DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+        export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
 
-    - name: Ad Click Detection Flow tests
+    - name: End to End tests
       run: |
-        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=adClick DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
+        export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.MAESTRO_CLOUD_API_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=${{ matrix.test-tag }} DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/
       
+  notify-failure:
+    name: Notify on failure
+    if: ${{ always() && contains(join(needs.*.result, ','), 'failure') && github.ref_name == 'main' }}
+    needs: [build-end-to-end-tests, end-to-end-tests]
+    runs-on: ubuntu-latest
+
+    steps:
     - name: Create Asana task when workflow failed
       if: ${{ failure() }}
       run: |
@@ -76,13 +110,3 @@ jobs:
           --header "Authorization: Bearer ${{ secrets.ASANA_ACCESS_TOKEN }}" \
           --header "Content-Type: application/json" \
           --data ' { "data": { "name": "GH Workflow Failure - End to end tests", "workspace": "${{ vars.GH_ASANA_WORKSPACE_ID }}", "projects": [ "${{ vars.GH_ASANA_IOS_APP_PROJECT_ID }}" ], "notes" : "The end to end workflow has failed. See https://github.com/duckduckgo/iOS/actions/runs/${{ github.run_id }}. For instructions on how to handle the failure(s), check https://app.asana.com/0/0/1206423571874502/f" } }'
-
-    - name: Upload logs when workflow failed
-      uses: actions/upload-artifact@v4
-      if: failure() || cancelled()
-      with:
-        name: BuildLogs
-        path: |
-          xcodebuild.log
-          DerivedData/Logs/Test/*.xcresult
-        retention-days: 7

.maestro/data_clearing_tests/02_duckduckgo_settings.yml

@@ -17,7 +17,7 @@ tags:
 - pressKey: Enter
 
 # Change settings
-- tapOn: "Safe search: moderate"
+- tapOn: "Safe search"
 - tapOn: "Off"
 
 # Fire Button - twice, just to be sure

.maestro/release_tests/password-autofill.yaml

@@ -45,12 +45,8 @@ tags:
 - tapOn: "Close"
 
 # Validate standard form
-- runFlow:
-    file: ../shared/delay.yaml
 - assertVisible:
     id: "searchEntry"
-- runFlow:
-    file: ../shared/delay.yaml
 - tapOn:
     id: "searchEntry"
 - runFlow:
@@ -71,11 +67,7 @@ tags:
 # Validate multistep form
 - tapOn: 
     id: "searchEntry"
-- runFlow:
-    file: ../shared/delay.yaml
 - inputText: "https://privacy-test-pages.site/autofill/autoprompt/3-multistep-form.html"
-- runFlow:
-    file: ../shared/delay.yaml
 - pressKey: Enter
 - runFlow:
     file: ../shared/delay.yaml