functional-test.yml

name: Functional Tests

on:
  pull_request:
    branches:
    - master
    paths-ignore:
    - 'docs/**'
    - '*.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CI: true
  SHOW_XCODE_LOG: true
  APPIUM_TEST_SERVER_PORT: '4567'
  APPIUM_TEST_SERVER_HOST: '127.0.0.1'

jobs:
  build:

    # https://github.com/actions/runner-images/tree/main/images/macos
    strategy:
      matrix:
        e2eRoot:
        - basic
        - device
        - driver
        - web
        - long
        xcodeVersion: ['14.3.1', '15.4']
        include:
        - xcodeVersion: '14.3.1'
          iosVersion: '16.4'
          deviceName: 'iPhone 14'
          tvosVersion: '16.4'
          tvosDeviceName: 'Apple TV'
          platform: macos-13
        - xcodeVersion: '15.4'
          iosVersion: '17.5'
          deviceName: 'iPhone 15'
          skipTvOs: true
          platform: macos-14
      fail-fast: false

    runs-on: ${{ matrix.platform }}

    name: e2e
    steps:
    - uses: actions/checkout@v4

    - name: Install Node.js
      uses: actions/setup-node@v3
      with:
        node-version: 'lts/*'

    - name: Select Xcode
      uses: maxim-lobanov/setup-xcode@v1
      with:
        xcode-version: "${{ matrix.xcodeVersion }}"

    - run: xcrun simctl list devices available
      name: List Installed Simulators
    - run: |
        brew install xq
        xcrun simctl list runtimes
        xcrun --sdk iphonesimulator --show-sdk-version
      name: List Runtimes
    - name: Prepare iOS simulator
      id: prepareSimulator
      uses: futureware-tech/simulator-action@v3
      with:
        model: "${{ matrix.deviceName }}"
        os_version: "${{ matrix.iosVersion }}"
        shutdown_after_job: false
    - run: xcrun simctl bootstatus ${{ steps.prepareSimulator.outputs.udid }} -b
      name: Wait for Simulator to finish booting

    - run: |
        npm install --no-save mjpeg-consumer
        npm install mocha-multi-reporters --save-dev
        export cwd=$(pwd)
        pushd "$cwd"
        cd ~
        npm install -g appium
        appium driver install --source=local "$cwd"
        appium driver doctor xcuitest
        appium driver run xcuitest build-wda --sdk=${{ matrix.iosVersion }} --name="${{ matrix.deviceName }}"
        echo "Starting Appium server on $APPIUM_TEST_SERVER_HOST:$APPIUM_TEST_SERVER_PORT"
        nohup appium server \
          --port=$APPIUM_TEST_SERVER_PORT \
          --address=$APPIUM_TEST_SERVER_HOST \
          --relaxed-security \
          --keep-alive-timeout 1200 2>&1 > "$cwd/appium.log" &
        sleep 5
        popd
        cat appium.log
      name: Prepare the server

    - run: |
        if ! nc -z $APPIUM_TEST_SERVER_HOST $APPIUM_TEST_SERVER_PORT; then
          echo "Appium server is still not listening at $APPIUM_TEST_SERVER_HOST:$APPIUM_TEST_SERVER_PORT"
          cat appium.log
          exit 1
        fi

        export DEVICE_NAME="${{ matrix.deviceName }}"
        export PLATFORM_VERSION="${{ matrix.iosVersion }}"

        export RESULTS_XML=test-results.xml
        echo "{\"reporterEnabled\": \"spec, xunit\", \"xunitReporterOptions\": {\"output\": \"$RESULTS_XML\"}}" > reporter_config.json
        ARGS=("./test/functional/${{ matrix.e2eRoot }}" --exit --timeout 10m --reporter mocha-multi-reporters --reporter-options configFile=reporter_config.json)
        if [[ "${{ matrix.e2eRoot }}" == basic ]]; then
          ARGS+=(--grep XCUITestDriver)
        fi
        if ! npx mocha "${ARGS[@]}"; then
          tests=$(cat "$RESULTS_XML" | xq --xpath '//testsuite/@tests')
          errors=$(cat "$RESULTS_XML" | xq --xpath '//testsuite/@errors')
          skipped=$(cat "$RESULTS_XML" | xq --xpath '//testsuite/@skipped')
          failures=$(cat "$RESULTS_XML" | xq --xpath '//testsuite/@failures')
          threshold=$(( (failures + errors) * 100 / (tests - skipped) ))
          cat "$RESULTS_XML"
          if [[ $threshold -gt 8 ]]; then
            echo "${threshold}% of tests failed"
            exit 1
          else
            echo "${threshold}% of tests failed. This is (probably) fine"
          fi
        fi
      name: Run functional tests

    - name: Save server output
      if: ${{ always() }}
      uses: actions/upload-artifact@master
      with:
        name: appium_e2eRoot_${{ matrix.e2eRoot }}_xCodev${{ matrix.xcodeVersion }}_iOSv${{ matrix.iosVersion }}.log
        path: appium.log