Merge remote-tracking branch 'upstream/master' into pr-escalade-esv-2019 #2943
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: selfdrive | |
| on: | |
| push: | |
| branches-ignore: | |
| - 'testing-closet*' | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| env: | |
| BASE_IMAGE: openpilot-base | |
| CL_BASE_IMAGE: openpilot-base-cl | |
| DOCKER_REGISTRY: ghcr.io/commaai | |
| AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} | |
| DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} | |
| BUILD: | | |
| DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . | |
| RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c | |
| BUILD_CL: | | |
| DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl . | |
| RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c | |
| UNIT_TEST: coverage run --append -m unittest discover | |
| jobs: | |
| build_release: | |
| name: build release | |
| runs-on: ubuntu-20.04 | |
| env: | |
| STRIPPED_DIR: /tmp/releasepilot | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Build devel | |
| run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh | |
| - uses: ./.github/workflows/setup | |
| - name: Check submodules | |
| if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' | |
| run: release/check-submodules.sh | |
| - name: Build openpilot and run checks | |
| run: | | |
| cd $STRIPPED_DIR | |
| ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py" | |
| - name: Run tests | |
| run: | | |
| cd $STRIPPED_DIR | |
| ${{ env.RUN }} "release/check-dirty.sh && \ | |
| python -m unittest discover selfdrive/car" | |
| - name: pre-commit | |
| run: | | |
| cd $GITHUB_WORKSPACE | |
| cp .pre-commit-config.yaml $STRIPPED_DIR | |
| cp .pylintrc $STRIPPED_DIR | |
| cp mypy.ini $STRIPPED_DIR | |
| cp pyproject.toml $STRIPPED_DIR | |
| cp poetry.lock $STRIPPED_DIR | |
| cd $STRIPPED_DIR | |
| ${{ env.RUN }} "pre-commit run --all" | |
| build_all: | |
| name: build all | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| with: | |
| save-cache: true | |
| - name: Build openpilot with all flags | |
| run: ${{ env.RUN }} "scons -j$(nproc) --extras && release/check-dirty.sh" | |
| - name: Cleanup scons cache | |
| run: | | |
| ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ | |
| scons -j$(nproc) --cache-populate" | |
| build_mac: | |
| name: build macos | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Determine pre-existing Homebrew packages | |
| if: steps.dependency-cache.outputs.cache-hit != 'true' | |
| run: | | |
| echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV | |
| brew list --formula -1 >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - name: Restore scons cache | |
| id: scons-restore-cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: /tmp/scons_cache | |
| key: macos_scons-${{ github.sha }} | |
| restore-keys: macos_scons- | |
| - name: Cache dependencies | |
| id: dependency-cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/github_brew_cache_entries.txt | |
| ~/.pyenv | |
| ~/Library/Caches/pypoetry | |
| /usr/local/Cellar | |
| /usr/local/opt | |
| /usr/local/Caskroom/gcc-arm-* | |
| /opt/homebrew/Cellar | |
| /opt/homebrew/opt | |
| /opt/homebrew/Caskroom/gcc-arm-* | |
| /Applications/ArmGNUToolchain/*/*/* | |
| key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }} | |
| restore-keys: macos_deps- | |
| - name: Brew link restored dependencies | |
| run: | | |
| if [ -f ~/github_brew_cache_entries.txt ]; then | |
| brew link --force --overwrite $(cat ~/github_brew_cache_entries.txt) # `--force` for keg-only packages | |
| if [ -d /Applications/ArmGNUToolchain ]; then # link gcc-arm-embedded manually | |
| GCC_TOOLCHAIN="$(echo /Applications/ArmGNUToolchain/**/**/bin)" | |
| echo "$GCC_TOOLCHAIN" >> $GITHUB_PATH | |
| fi | |
| else | |
| echo "Cache entries not found" | |
| fi | |
| - name: Install dependencies | |
| if: steps.dependency-cache.outputs.cache-hit != 'true' | |
| run: ./tools/mac_setup.sh | |
| - name: Build openpilot | |
| run: | | |
| source tools/openpilot_env.sh | |
| poetry run scons -j$(nproc) | |
| - name: Pre Cache - Cleanup scons cache | |
| if: github.ref == 'refs/heads/master' | |
| run: | | |
| source tools/openpilot_env.sh | |
| rm -rf /tmp/scons_cache/* | |
| poetry run scons -j$(nproc) --cache-populate | |
| - name: Save scons cache | |
| id: scons-save-cache | |
| uses: actions/cache/save@v3 | |
| if: github.ref == 'refs/heads/master' | |
| with: | |
| path: /tmp/scons_cache | |
| key: macos_scons-${{ github.sha }} | |
| - name: Pre Cache - Remove pre-existing Homebrew packages | |
| if: steps.dependency-cache.outputs.cache-hit != 'true' | |
| run: | | |
| new_cellar=$(brew list --formula -1) | |
| comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do | |
| if [[ $pkg != "zstd" ]]; then # caching step needs zstd | |
| rm -rf "$(brew --cellar)/$pkg" | |
| fi | |
| done | |
| comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt | |
| # .fseventsd directory causes permission errors in dep caching step | |
| # its used by the system to observe changes within the directory - toolchain works without it, just remove it | |
| sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd | |
| docker_push: | |
| name: docker push | |
| runs-on: ubuntu-20.04 | |
| if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' | |
| needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Build Docker image | |
| run: eval "$BUILD" | |
| - name: Push to container registry | |
| run: | | |
| $DOCKER_LOGIN | |
| docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest | |
| - name: Build CL Docker image | |
| run: eval "$BUILD_CL" | |
| - name: Push to container registry | |
| run: | | |
| $DOCKER_LOGIN | |
| docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest | |
| static_analysis: | |
| name: static analysis | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Build Docker image | |
| run: eval "$BUILD" | |
| - name: pre-commit | |
| run: ${{ env.RUN }} "pre-commit run --all" | |
| valgrind: | |
| name: valgrind | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| - name: Build openpilot | |
| run: ${{ env.RUN }} "scons -j$(nproc)" | |
| - name: Run valgrind | |
| run: | | |
| ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py" | |
| - name: Print logs | |
| if: always() | |
| run: cat selfdrive/test/valgrind_logs.txt | |
| unit_tests: | |
| name: unit tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| - name: Build openpilot | |
| run: ${{ env.RUN }} "scons -j$(nproc)" | |
| - name: Run unit tests | |
| run: | | |
| ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \ | |
| $UNIT_TEST common && \ | |
| $UNIT_TEST opendbc/can && \ | |
| $UNIT_TEST selfdrive/boardd && \ | |
| $UNIT_TEST selfdrive/controls && \ | |
| $UNIT_TEST selfdrive/monitoring && \ | |
| $UNIT_TEST system/loggerd && \ | |
| $UNIT_TEST selfdrive/car && \ | |
| $UNIT_TEST selfdrive/locationd && \ | |
| $UNIT_TEST system/tests && \ | |
| $UNIT_TEST system/ubloxd && \ | |
| selfdrive/locationd/test/_test_locationd_lib.py && \ | |
| ./system/ubloxd/tests/test_glonass_runner && \ | |
| $UNIT_TEST selfdrive/athena && \ | |
| $UNIT_TEST selfdrive/thermald && \ | |
| $UNIT_TEST system/hardware/tici && \ | |
| $UNIT_TEST tools/lib/tests && \ | |
| ./selfdrive/ui/tests/create_test_translations.sh && \ | |
| QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ | |
| ./selfdrive/ui/tests/test_translations.py && \ | |
| ./common/tests/test_util && \ | |
| ./common/tests/test_swaglog && \ | |
| ./selfdrive/boardd/tests/test_boardd_usbprotocol && \ | |
| ./system/loggerd/tests/test_logger &&\ | |
| ./system/proclogd/tests/test_proclog && \ | |
| ./tools/replay/tests/test_replay && \ | |
| ./tools/cabana/tests/test_cabana && \ | |
| ./system/camerad/test/ae_gray_test && \ | |
| ./selfdrive/test/process_replay/test_fuzzy.py && \ | |
| coverage xml" | |
| - name: "Upload coverage to Codecov" | |
| uses: codecov/codecov-action@v2 | |
| process_replay: | |
| name: process replay | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| - name: Cache test routes | |
| id: dependency-cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: /tmp/comma_download_cache | |
| key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }} | |
| - name: Build openpilot | |
| run: | | |
| ${{ env.RUN }} "scons -j$(nproc)" | |
| - name: Run replay | |
| run: | | |
| ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ | |
| coverage xml" | |
| - name: Print diff | |
| id: print-diff | |
| if: always() | |
| run: cat selfdrive/test/process_replay/diff.txt | |
| - uses: actions/upload-artifact@v2 | |
| if: always() | |
| continue-on-error: true | |
| with: | |
| name: process_replay_diff.txt | |
| path: selfdrive/test/process_replay/diff.txt | |
| - name: Upload reference logs | |
| if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }} | |
| run: | | |
| ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only" | |
| - name: "Upload coverage to Codecov" | |
| uses: codecov/codecov-action@v2 | |
| test_modeld: | |
| name: model tests | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| - name: Build Docker image | |
| # Sim docker is needed to get the OpenCL drivers | |
| run: eval "$BUILD_CL" | |
| - name: Build openpilot | |
| run: | | |
| ${{ env.RUN }} "scons -j$(nproc)" | |
| - name: Run model replay with ONNX | |
| run: | | |
| ${{ env.RUN_CL }} "ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \ | |
| coverage xml" | |
| - name: Run unit tests | |
| run: | | |
| ${{ env.RUN_CL }} "$UNIT_TEST selfdrive/modeld && \ | |
| coverage xml" | |
| - name: "Upload coverage to Codecov" | |
| uses: codecov/codecov-action@v2 | |
| test_longitudinal: | |
| name: longitudinal | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| - name: Build openpilot | |
| run: | | |
| ${{ env.RUN }} "scons -j$(nproc)" | |
| - name: Test longitudinal | |
| run: | | |
| ${{ env.RUN }} "mkdir -p selfdrive/test/out && \ | |
| cd selfdrive/test/longitudinal_maneuvers && \ | |
| coverage run ./test_longitudinal.py && \ | |
| coverage xml" | |
| - name: "Upload coverage to Codecov" | |
| uses: codecov/codecov-action@v2 | |
| - uses: actions/upload-artifact@v2 | |
| if: always() | |
| continue-on-error: true | |
| with: | |
| name: longitudinal | |
| path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/ | |
| test_cars: | |
| name: cars | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: [0, 1, 2, 3, 4] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - uses: ./.github/workflows/setup | |
| - name: Cache test routes | |
| id: dependency-cache | |
| uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b | |
| with: | |
| path: /tmp/comma_download_cache | |
| key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }} | |
| - name: Build openpilot | |
| run: ${{ env.RUN }} "scons -j$(nproc)" | |
| - name: Test car models | |
| run: | | |
| ${{ env.RUN }} "coverage run -m pytest selfdrive/car/tests/test_models.py && \ | |
| coverage xml && \ | |
| chmod -R 777 /tmp/comma_download_cache" | |
| env: | |
| NUM_JOBS: 5 | |
| JOB_ID: ${{ matrix.job }} | |
| - name: "Upload coverage to Codecov" | |
| uses: codecov/codecov-action@v2 | |
| car_docs_diff: | |
| name: PR comments | |
| runs-on: ubuntu-20.04 | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.base.ref }} | |
| - uses: ./.github/workflows/setup | |
| - name: Get base car info | |
| run: | | |
| ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info" | |
| sudo chown -R $USER:$USER ${{ github.workspace }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Save car docs diff | |
| id: save_diff | |
| run: | | |
| ${{ env.RUN }} "scons -j$(nproc)" | |
| output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info") || true | |
| output="${output//$'\n'/'%0A'}" | |
| echo "::set-output name=diff::$output" | |
| - name: Find comment | |
| if: ${{ env.AZURE_TOKEN != '' }} | |
| uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d | |
| id: fc | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-includes: This PR makes changes to | |
| - name: Update comment | |
| if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }} | |
| uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body: "${{ steps.save_diff.outputs.diff }}" | |
| edit-mode: replace | |
| - name: Delete comment | |
| if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }} | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| github.rest.issues.deleteComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: ${{ steps.fc.outputs.comment-id }} | |
| }) |