selfdrive_tests.yaml

name: selfdrive

on:
  push:
    branches:
      - master
  pull_request:

concurrency:
  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  PYTHONWARNINGS: error
  BASE_IMAGE: openpilot-base
  CL_BASE_IMAGE: openpilot-base-cl
  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c

  BUILD_CL: selfdrive/test/docker_build.sh cl

  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c

  UNIT_TEST: coverage run --append -m unittest discover
  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5

jobs:
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build devel
      timeout-minutes: 1
      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
    - uses: ./.github/workflows/setup-with-retry
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      timeout-minutes: 1
      run: release/check-submodules.sh
    - name: Build openpilot and run checks
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py"
    - name: Run tests
      timeout-minutes: 2
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "release/check-dirty.sh && \
                        python -m unittest discover selfdrive/car"
    - name: pre-commit
      timeout-minutes: 3
      run: |
        cd $GITHUB_WORKSPACE
        cp .pre-commit-config.yaml $STRIPPED_DIR
        cp pyproject.toml $STRIPPED_DIR
        cp poetry.lock $STRIPPED_DIR
        cd $STRIPPED_DIR
        ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"

  build:
    strategy:
      matrix:
        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
        cache_key_prefix: scons_${{ matrix.arch }}
    - uses: ./.github/workflows/compile-openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
      with:
        cache_key_prefix: scons_${{ matrix.arch }}

  build_mac:
    name: build macos
    runs-on: macos-latest
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Determine pre-existing Homebrew packages
      if: steps.dependency-cache.outputs.cache-hit != 'true'
      run: |
        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
        brew list --formula -1 >> $GITHUB_ENV
        echo 'EOF' >> $GITHUB_ENV
    - name: Restore scons cache
      id: scons-restore-cache
      uses: actions/cache/restore@v3
      with:
        path: /tmp/scons_cache
        key: macos_scons-${{ github.sha }}
        restore-keys: macos_scons-
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v3
      with:
        path: |
          .env
          .venv
          ~/github_brew_cache_entries.txt
          ~/.pyenv
          ~/Library/Caches/pypoetry
          /usr/local/Cellar
          /usr/local/opt
          /usr/local/Caskroom/gcc-arm-*
          /opt/homebrew/Cellar
          /opt/homebrew/opt
          /opt/homebrew/Caskroom/gcc-arm-*
          /Applications/ArmGNUToolchain/*/*/*
        key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'tools/install_python_dependencies.sh', 'poetry.lock') }}
        restore-keys: macos_deps-
    - name: Brew link restored dependencies
      run: |
        if [ -f ~/github_brew_cache_entries.txt ]; then
          brew link --force --overwrite $(cat ~/github_brew_cache_entries.txt) # `--force` for keg-only packages
          if [ -d /Applications/ArmGNUToolchain ]; then # link gcc-arm-embedded manually
            GCC_TOOLCHAIN="$(echo /Applications/ArmGNUToolchain/**/**/bin)"
            echo "$GCC_TOOLCHAIN" >> $GITHUB_PATH
          fi
        else
          echo "Cache entries not found"
        fi
    - name: Install dependencies
      if: steps.dependency-cache.outputs.cache-hit != 'true'
      run: SKIP_PROMPT=1 ./tools/mac_setup.sh
      env:
        # package install has DeprecationWarnings
        PYTHONWARNINGS: default
    - name: Build openpilot
      run: |
        eval "$(pyenv init --path)"
        poetry run scons -j$(nproc)
    - name: Run tests
      run: |
        eval "$(pyenv init --path)"
        poetry run tools/plotjuggler/test_plotjuggler.py
    - name: Pre Cache - Cleanup scons cache
      if: github.ref == 'refs/heads/master'
      run: |
        rm -rf /tmp/scons_cache/*
        eval "$(pyenv init --path)"
        poetry run scons -j$(nproc) --cache-populate
    - name: Save scons cache
      id: scons-save-cache
      uses: actions/cache/save@v3
      if: github.ref == 'refs/heads/master'
      with:
        path: /tmp/scons_cache
        key: macos_scons-${{ github.sha }}
    - name: Pre Cache - Remove pre-existing Homebrew packages
      if: steps.dependency-cache.outputs.cache-hit != 'true'
      run: |
        new_cellar=$(brew list --formula -1)
        exceptions="zstd lz4 xz" # caching step needs zstd
        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
          if [[ " $exceptions " != *" $pkg "* ]]; then
            rm -rf "$(brew --cellar)/$pkg"
          fi
        done
        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
        # .fseventsd directory causes permission errors in dep caching step
        # its used by the system to observe changes within the directory - toolchain works without it, just remove it
        sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd

  docker_push:
    name: docker push
    strategy:
      matrix:
        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Setup to push to repo
      run: |
        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
        $DOCKER_LOGIN
    - uses: ./.github/workflows/setup-with-retry
      with:
        git-lfs: false
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Build and push CL Docker image
      if: matrix.arch == 'x86_64'
      run: |
        unset TARGET_ARCHITECTURE
        eval "$BUILD_CL"

  docker_push_multiarch:
    name: docker push multiarch tag
    runs-on: ubuntu-20.04
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: [docker_push]
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: false
    - name: Setup docker
      run: |
        $DOCKER_LOGIN
    - name: Merge x64 and arm64 tags
      run: |
        export PUSH_IMAGE=true
        selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64

  static_analysis:
    name: static analysis
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: pre-commit
      timeout-minutes: 4
      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run valgrind
      timeout-minutes: 1
      run: |
        ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py"
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt

  unit_tests:
    name: unit tests
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run unit tests
      timeout-minutes: 15
      run: |
        ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
                        $PYTEST -n auto --dist=loadscope --timeout 30 -o cpp_files=test_* && \
                        ./selfdrive/ui/tests/create_test_translations.sh && \
                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                        ./selfdrive/ui/tests/test_translations.py && \
                        ./system/camerad/test/ae_gray_test && \
                        ./selfdrive/test/process_replay/test_fuzzy.py"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3

  process_replay:
    name: process replay
    runs-on: ${{ ((github.event.pull_request.head.repo.full_name == 'commaai/openpilot') || (github.repository == 'commaai/openpilot')) && 'buildjet-8vcpu-ubuntu-2004' || 'ubuntu-20.04' }}
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v3
      with:
        path: .ci_cache/comma_download_cache
        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run replay
      timeout-minutes: 30
      run: |
        ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        chmod -R 777 /tmp/comma_download_cache && \
                        coverage xml"
    - name: Print diff
      id: print-diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt
    - name: Upload reference logs
      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3

  test_modeld:
    name: model tests
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build base Docker image
      run: eval "$BUILD"
    - name: Build Docker image
      # Sim docker is needed to get the OpenCL drivers
      run: eval "$BUILD_CL"
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
    - name: Run model replay with ONNX
      timeout-minutes: 3
      run: |
        ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \
                           ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
                           coverage xml"
    - name: Run unit tests
      timeout-minutes: 4
      run: |
        ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \
                           $UNIT_TEST selfdrive/modeld && \
                           coverage xml"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3

  test_cars:
    name: cars
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        job: [0, 1, 2, 3, 4]
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v3
      with:
        path: .ci_cache/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Test car models
      timeout-minutes: 25
      run: |
        ${{ env.RUN }} "$PYTEST -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
        NUM_JOBS: 5
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v3

  car_docs_diff:
    name: PR comments
    runs-on: ubuntu-20.04
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
          ref: ${{ github.event.pull_request.base.ref }}
      - uses: ./.github/workflows/setup-with-retry
      - name: Get base car info
        run: |
          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
          sudo chown -R $USER:$USER ${{ github.workspace }}
      - uses: actions/checkout@v3
        with:
          submodules: true
          path: current
      - name: Save car docs diff
        id: save_diff
        run: |
          cd current
          ${{ env.RUN }} "scons -j$(nproc)"
          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
          output="${output//$'\n'/'%0A'}"
          echo "::set-output name=diff::$output"
      - name: Find comment
        if: ${{ env.AZURE_TOKEN != '' }}
        uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-includes: This PR makes changes to
      - name: Update comment
        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
        uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: "${{ steps.save_diff.outputs.diff }}"
          edit-mode: replace
      - name: Delete comment
        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
        uses: actions/github-script@v6
        with:
          script: |
            github.rest.issues.deleteComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.fc.outputs.comment-id }}
            })