remove can filter submodule #226

Workflow file for this run

.github/workflows/selfdrive_tests.yaml at 53c70c4

	name: selfdrive
	on:
	push:
	branches-ignore:
	- 'testing-closet*'
	pull_request:

	env:
	BASE_IMAGE: openpilot-base
	DOCKER_REGISTRY: ghcr.io/commaai

	DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
	BUILD: \|
	docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) \|\| true
	docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest \|\| true
	docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

	RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
	UNIT_TEST: coverage run --append -m unittest discover

	jobs:
	# TODO: once actions/cache supports read only mode, use the cache for all jobs
	build_release:
	name: build release
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	env:
	STRIPPED_DIR: tmppilot
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Check submodules
	if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
	run: release/check-submodules.sh
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Strip non-release files
	run: \|
	mkdir $STRIPPED_DIR
	cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
	cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
	cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
	cp Dockerfile.openpilot_base $STRIPPED_DIR

	# need this to build on x86
	cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Build openpilot and run checks
	run: \|
	cd $STRIPPED_DIR
	${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
	python -m unittest discover selfdrive/car"
	- name: Cleanup scons cache
	run: \|
	cd $STRIPPED_DIR
	${{ env.RUN }} "scons -j$(nproc) && \
	rm -rf /tmp/scons_cache/* && \
	scons -j$(nproc) --cache-populate"

	#build_mac:
	# name: build macos
	# runs-on: macos-latest
	# timeout-minutes: 60
	# steps:
	# - uses: actions/checkout@v2
	# with:
	# submodules: true
	# - name: Determine pre-existing Homebrew packages
	# if: steps.dependency-cache.outputs.cache-hit != 'true'
	# run: \|
	# echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
	# ls -1 /usr/local/Cellar >> $GITHUB_ENV
	# echo 'EOF' >> $GITHUB_ENV
	# - name: Cache dependencies
	# id: dependency-cache
	# uses: actions/cache@v2
	# with:
	# path: \|
	# ~/.pyenv
	# ~/.local/share/virtualenvs/
	# /usr/local/Cellar
	# ~/github_brew_cache_entries.txt
	# /tmp/scons_cache
	# key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
	# restore-keys: macos-
	# - name: Brew link restored dependencies
	# run: \|
	# if [ -f ~/github_brew_cache_entries.txt ]; then
	# while read pkg; do
	# brew link --force "$pkg" # `--force` for keg-only packages
	# done < ~/github_brew_cache_entries.txt
	# else
	# echo "Cache entries not found"
	# fi
	# - name: Install dependencies
	# run: ./tools/mac_setup.sh
	# - name: Build openpilot
	# run: \|
	# source tools/openpilot_env.sh
	# pipenv run selfdrive/manager/build.py
	#
	# # cleanup scons cache
	# rm -rf /tmp/scons_cache/
	# pipenv run scons -j$(nproc) --cache-populate
	# - name: Remove pre-existing Homebrew packages for caching
	# if: steps.dependency-cache.outputs.cache-hit != 'true'
	# run: \|
	# cd /usr/local/Cellar
	# new_cellar=$(ls -1)
	# comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") \| while read pkg; do
	# if [[ $pkg != "zstd" ]]; then # caching step needs zstd
	# rm -rf "$pkg"
	# fi
	# done
	# comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") \| tee ~/github_brew_cache_entries.txt

	build_webcam:
	name: build webcam
	runs-on: ubuntu-20.04
	timeout-minutes: 90
	env:
	IMAGE_NAME: openpilotwebcamci
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build Docker image
	run: \|
	eval "$BUILD"
	docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest \|\| true
	docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
	- name: Build openpilot
	run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
	- name: Push to container registry
	if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
	run: \|
	$DOCKER_LOGIN
	docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest

	docker_push:
	name: docker push
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
	needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Push to container registry
	run: \|
	$DOCKER_LOGIN
	docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest

	static_analysis:
	name: static analysis
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build Docker image
	run: eval "$BUILD"
	- name: pre-commit
	run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all"

	valgrind:
	name: valgrind
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache dependencies
	id: dependency-cache
	uses: actions/cache@v2
	with:
	path: /tmp/comma_download_cache
	key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Run valgrind
	run: \|
	${{ env.RUN }} "scons -j$(nproc) && \
	FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
	- name: Print logs
	if: always()
	run: cat selfdrive/test/valgrind_logs.txt

	unit_tests:
	name: unit tests
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- name: Get current date
	id: date
	run: echo "::set-output name=time::$(date +'%s')"
	- name: Output timestamp
	run: echo $TIMESTAMP
	env:
	TIMESTAMP: ${{ steps.date.outputs.time }}
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' \|\| github.repository != 'commaai/openpilot' }}
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
	restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Run unit tests
	run: \|
	${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
	scons -j$(nproc) --test && \
	$UNIT_TEST common && \
	$UNIT_TEST opendbc/can && \
	$UNIT_TEST selfdrive/boardd && \
	$UNIT_TEST selfdrive/controls && \
	$UNIT_TEST selfdrive/monitoring && \
	$UNIT_TEST selfdrive/loggerd && \
	$UNIT_TEST selfdrive/car && \
	$UNIT_TEST selfdrive/locationd && \
	$UNIT_TEST selfdrive/athena && \
	$UNIT_TEST selfdrive/thermald && \
	$UNIT_TEST selfdrive/hardware/tici && \
	$UNIT_TEST tools/lib/tests && \
	./selfdrive/boardd/tests/test_boardd_usbprotocol && \
	./selfdrive/common/tests/test_util && \
	./selfdrive/common/tests/test_swaglog && \
	./selfdrive/loggerd/tests/test_logger &&\
	./selfdrive/proclogd/tests/test_proclog && \
	./selfdrive/ui/replay/tests/test_replay && \
	./selfdrive/camerad/test/ae_gray_test && \
	coverage xml"
	- name: "Upload coverage to Codecov"
	uses: codecov/codecov-action@v2

	process_replay:
	name: process replay
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache dependencies
	id: dependency-cache
	uses: actions/cache@v2
	with:
	path: /tmp/comma_download_cache
	key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Run replay
	run: \|
	${{ env.RUN }} "scons -j$(nproc) && \
	FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py && \
	coverage xml"
	- name: "Upload coverage to Codecov"
	uses: codecov/codecov-action@v2
	- name: Print diff
	if: always()
	run: cat selfdrive/test/process_replay/diff.txt
	- uses: actions/upload-artifact@v2
	if: always()
	continue-on-error: true
	with:
	name: process_replay_diff.txt
	path: selfdrive/test/process_replay/diff.txt

	#model_replay:
	# name: model replay
	# runs-on: ubuntu-20.04
	# timeout-minutes: 50
	# steps:
	# - uses: actions/checkout@v2
	# with:
	# submodules: true
	# - name: Build Docker image
	# run: eval "$BUILD"
	# - name: Run replay
	# run: \|
	# ${{ env.RUN }} "scons -j$(nproc) && \
	# selfdrive/test/process_replay/model_replay.py"

	test_longitudinal:
	name: longitudinal
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Test longitudinal
	run: \|
	${{ env.RUN }} "mkdir -p selfdrive/test/out && \
	scons -j$(nproc) && \
	cd selfdrive/test/longitudinal_maneuvers && \
	coverage run ./test_longitudinal.py && \
	coverage xml"
	- name: "Upload coverage to Codecov"
	uses: codecov/codecov-action@v2
	- uses: actions/upload-artifact@v2
	if: always()
	continue-on-error: true
	with:
	name: longitudinal
	path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/

	test_cars:
	name: cars
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	strategy:
	fail-fast: false
	matrix:
	job: [0, 1, 2, 3]
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache dependencies
	id: dependency-cache
	uses: actions/cache@v2
	with:
	path: /tmp/comma_download_cache
	key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Test car models
	run: \|
	${{ env.RUN }} "scons -j$(nproc) --test && \
	FILEREADER_CACHE=1 coverage run -m pytest selfdrive/car/tests/test_models.py && \
	coverage xml && \
	chmod -R 777 /tmp/comma_download_cache"
	env:
	NUM_JOBS: 4
	JOB_ID: ${{ matrix.job }}
	- name: "Upload coverage to Codecov"
	uses: codecov/codecov-action@v2

	docs:
	name: build docs
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build docker container
	run: \|
	docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest \|\| true
	docker pull $DOCKER_REGISTRY/openpilot-docs:latest \|\| true
	DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
	- name: Push docker container
	if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
	run: \|
	$DOCKER_LOGIN
	docker push $DOCKER_REGISTRY/openpilot-docs:latest

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

remove can filter submodule #226

Workflow file

remove can filter submodule #226

Jobs

Run details

Workflow file for this run