Update tunes.py DP tune #338

Workflow file for this run

.github/workflows/selfdrive_tests.yaml at 9e58444

	name: selfdrive
	on:
	push:
	branches-ignore:
	- 'testing-closet*'
	pull_request:

	env:
	BASE_IMAGE: openpilot-base
	DOCKER_REGISTRY: ghcr.io/commaai

	DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
	BUILD: \|
	docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) \|\| true
	docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest \|\| true
	docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

	RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
	UNIT_TEST: coverage run --append -m unittest discover

	jobs:
	# TODO: once actions/cache supports read only mode, use the cache for all jobs
	build_release:
	name: build release
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	env:
	STRIPPED_DIR: tmppilot
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Check submodules
	if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
	run: release/check-submodules.sh
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Strip non-release files
	run: \|
	mkdir $STRIPPED_DIR
	cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
	cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
	cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
	cp Dockerfile.openpilot_base $STRIPPED_DIR

	# need this to build on x86
	cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Build openpilot and run checks
	run: \|
	cd $STRIPPED_DIR
	${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
	python -m unittest discover selfdrive/car"
	- name: Cleanup scons cache
	run: \|
	cd $STRIPPED_DIR
	${{ env.RUN }} "scons -j$(nproc) && \
	rm -rf /tmp/scons_cache/* && \
	scons -j$(nproc) --cache-populate"

	build_mac:
	name: build macos
	runs-on: macos-latest
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Determine pre-existing Homebrew packages
	if: steps.dependency-cache.outputs.cache-hit != 'true'
	run: \|
	echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
	ls -1 /usr/local/Cellar >> $GITHUB_ENV
	echo 'EOF' >> $GITHUB_ENV
	- name: Cache dependencies
	id: dependency-cache
	uses: actions/cache@v2
	with:
	path: \|
	~/.pyenv
	~/.local/share/virtualenvs/
	/usr/local/Cellar
	~/github_brew_cache_entries.txt
	/tmp/scons_cache
	key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
	restore-keys: macos-
	- name: Brew link restored dependencies
	run: \|
	if [ -f ~/github_brew_cache_entries.txt ]; then
	while read pkg; do
	brew link --force "$pkg" # `--force` for keg-only packages
	done < ~/github_brew_cache_entries.txt
	else
	echo "Cache entries not found"
	fi
	- name: Install dependencies
	run: ./tools/mac_setup.sh
	- name: Build openpilot
	run: \|
	source tools/openpilot_env.sh
	pipenv run selfdrive/manager/build.py

	# cleanup scons cache
	rm -rf /tmp/scons_cache/
	pipenv run scons -j$(nproc) --cache-populate
	- name: Remove pre-existing Homebrew packages for caching
	if: steps.dependency-cache.outputs.cache-hit != 'true'
	run: \|
	cd /usr/local/Cellar
	new_cellar=$(ls -1)
	comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") \| while read pkg; do
	if [[ $pkg != "zstd" ]]; then # caching step needs zstd
	rm -rf "$pkg"
	fi
	done
	comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") \| tee ~/github_brew_cache_entries.txt

	build_webcam:
	name: build webcam
	runs-on: ubuntu-20.04
	timeout-minutes: 90
	env:
	IMAGE_NAME: openpilotwebcamci
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build Docker image
	run: \|
	eval "$BUILD"
	docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest \|\| true
	docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
	- name: Build openpilot
	run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
	- name: Push to container registry
	if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
	run: \|
	$DOCKER_LOGIN
	docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest

	docker_push:
	name: docker push
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
	# needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Push to container registry
	run: \|
	$DOCKER_LOGIN
	docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest

	# static_analysis:
	# name: static analysis
	# runs-on: ubuntu-20.04
	# timeout-minutes: 50
	# steps:
	# - uses: actions/checkout@v2
	# with:
	# submodules: true
	# - name: Build Docker image
	# run: eval "$BUILD"
	# - name: pre-commit
	# run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all"

	valgrind:
	name: valgrind
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache dependencies
	id: dependency-cache
	uses: actions/cache@v2
	with:
	path: /tmp/comma_download_cache
	key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Run valgrind
	run: \|
	${{ env.RUN }} "scons -j$(nproc) && \
	FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
	- name: Print logs
	if: always()
	run: cat selfdrive/test/valgrind_logs.txt

	unit_tests:
	name: unit tests
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- name: Get current date
	id: date
	run: echo "::set-output name=time::$(date +'%s')"
	- name: Output timestamp
	run: echo $TIMESTAMP
	env:
	TIMESTAMP: ${{ steps.date.outputs.time }}
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' \|\| github.repository != 'commaai/openpilot' }}
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
	restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Run unit tests
	run: \|
	${{ env.RUN }} "scons -j$(nproc) --test && \
	coverage run selfdrive/test/test_fingerprints.py && \
	$UNIT_TEST common && \
	$UNIT_TEST opendbc/can && \
	$UNIT_TEST selfdrive/boardd && \
	$UNIT_TEST selfdrive/controls && \
	$UNIT_TEST selfdrive/monitoring && \
	$UNIT_TEST selfdrive/loggerd && \
	$UNIT_TEST selfdrive/car && \
	$UNIT_TEST selfdrive/locationd && \
	$UNIT_TEST selfdrive/athena && \
	$UNIT_TEST selfdrive/thermald && \
	$UNIT_TEST selfdrive/hardware/tici && \
	$UNIT_TEST tools/lib/tests && \
	./selfdrive/boardd/tests/test_boardd_usbprotocol && \
	./selfdrive/common/tests/test_util && \
	./selfdrive/loggerd/tests/test_logger &&\
	./selfdrive/proclogd/tests/test_proclog && \
	./selfdrive/ui/replay/tests/test_replay && \
	./selfdrive/camerad/test/ae_gray_test"
	- name: Upload coverage to Codecov
	run: bash <(curl -s https://codecov.io/bash) -v -F unit_tests

	# process_replay:
	# name: process replay
	# runs-on: ubuntu-20.04
	# timeout-minutes: 50
	# steps:
	# - uses: actions/checkout@v2
	# with:
	# submodules: true
	# - name: Cache dependencies
	# id: dependency-cache
	# uses: actions/cache@v2
	# with:
	# path: /tmp/comma_download_cache
	# key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
	# - name: Cache scons
	# id: scons-cache
	# # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	# uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	# env:
	# CACHE_SKIP_SAVE: true
	# with:
	# path: /tmp/scons_cache
	# key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	# restore-keys: \|
	# scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	# scons-
	# - name: Build Docker image
	# run: eval "$BUILD"
	# - name: Run replay
	# run: \|
	# ${{ env.RUN }} "scons -j$(nproc) && \
	# FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
	# - name: Upload coverage to Codecov
	# run: bash <(curl -s https://codecov.io/bash) -v -F process_replay
	# - name: Print diff
	# if: always()
	# run: cat selfdrive/test/process_replay/diff.txt
	# - uses: actions/upload-artifact@v2
	# if: always()
	# continue-on-error: true
	# with:
	# name: process_replay_diff.txt
	# path: selfdrive/test/process_replay/diff.txt

	#model_replay:
	# name: model replay
	# runs-on: ubuntu-20.04
	# timeout-minutes: 50
	# steps:
	# - uses: actions/checkout@v2
	# with:
	# submodules: true
	# - name: Build Docker image
	# run: eval "$BUILD"
	# - name: Run replay
	# run: \|
	# ${{ env.RUN }} "scons -j$(nproc) && \
	# selfdrive/test/process_replay/model_replay.py"

	test_longitudinal:
	name: longitudinal
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Test longitudinal
	run: \|
	${{ env.RUN }} "mkdir -p selfdrive/test/out && \
	scons -j$(nproc) && \
	cd selfdrive/test/longitudinal_maneuvers && \
	./test_longitudinal.py"
	- uses: actions/upload-artifact@v2
	if: always()
	continue-on-error: true
	with:
	name: longitudinal
	path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/

	test_cars:
	name: cars
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	strategy:
	fail-fast: false
	matrix:
	job: [0, 1, 2, 3]
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Cache dependencies
	id: dependency-cache
	uses: actions/cache@v2
	with:
	path: /tmp/comma_download_cache
	key: car_models-${{ hashFiles('selfdrive/test/test_models.py', 'selfdrive/test/test_routes.py') }}-${{ matrix.job }}
	- name: Cache scons
	id: scons-cache
	# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
	uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
	env:
	CACHE_SKIP_SAVE: true
	with:
	path: /tmp/scons_cache
	key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	restore-keys: \|
	scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
	scons-
	- name: Build Docker image
	run: eval "$BUILD"
	- name: Test car models
	run: \|
	${{ env.RUN }} "scons -j$(nproc) --test && \
	FILEREADER_CACHE=1 pytest selfdrive/test/test_models.py && \
	chmod -R 777 /tmp/comma_download_cache"
	env:
	NUM_JOBS: 4
	JOB_ID: ${{ matrix.job }}
	- name: Upload coverage to Codecov
	run: bash <(curl -s https://codecov.io/bash) -v -F test_car_models

	docs:
	name: build docs
	runs-on: ubuntu-20.04
	timeout-minutes: 50
	steps:
	- uses: actions/checkout@v2
	with:
	submodules: true
	- name: Build docker container
	run: \|
	docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest \|\| true
	docker pull $DOCKER_REGISTRY/openpilot-docs:latest \|\| true
	DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
	- name: Push docker container
	if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
	run: \|
	$DOCKER_LOGIN
	docker push $DOCKER_REGISTRY/openpilot-docs:latest

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Update tunes.py DP tune #338

Workflow file

Update tunes.py DP tune #338

Jobs

Run details

Workflow file for this run