Merge branch 'commaai:master' into master

lskov · Sep 5, 2023 · 9b71115 · 9b71115
2 parents 4f3bd75 + ed7a0bf
commit 9b71115
Show file tree

Hide file tree

Showing 246 changed files with 3,669 additions and 2,065 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -19,5 +19,16 @@
     "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
     "--volume=/tmp/devcontainer_scons_cache:/tmp/scons_cache",
     "--shm-size=1G"
-  ]
+  ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-vscode.cpptools",
+        "ms-toolsai.jupyter",
+        "guyskk.language-cython",
+        "lharri73.dbc"
+      ]
+    }
+  }
 }
diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml
@@ -7,7 +7,7 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
 
 jobs:
   badges:

diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml
@@ -5,20 +5,16 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
   build_prebuilt:
     name: build prebuilt
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     env:
-      IMAGE_NAME: openpilot-prebuilt
+      PUSH_IMAGE: true
     steps:
     - name: Wait for green check mark
       if: ${{ github.event_name != 'workflow_dispatch' }}
@@ -31,13 +27,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Build Docker image
-      run: |
-        eval "$BUILD"
-        DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
-    - name: Push to container registry
+    - name: Build and Push docker image
       run: |
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
-        docker tag $DOCKER_REGISTRY/$IMAGE_NAME:latest $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA
+        eval "$BUILD"
diff --git a/.github/workflows/repo.yml → .github/workflows/repo-maintenance.yaml b/.github/workflows/repo.yml → .github/workflows/repo-maintenance.yaml
@@ -1,18 +1,22 @@
-name: repo
+name: repo maintenance
 
 on:
   schedule:
     - cron: "0 15 * * 2"
   workflow_dispatch:
 
 jobs:
-  pre-commit-autoupdate:
-    name: pre-commit autoupdate
+  updates:
+    name: updates
     runs-on: ubuntu-20.04
     container:
       image: ghcr.io/commaai/openpilot-base:latest
     steps:
     - uses: actions/checkout@v3
+    - name: poetry lock
+      run: |
+        pip install poetry
+        poetry lock
     - name: pre-commit autoupdate
       run: |
         git config --global --add safe.directory '*'
@@ -21,8 +25,8 @@ jobs:
       uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
       with:
         token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
-        commit-message: Update pre-commit hook versions
-        title: 'pre-commit: autoupdate hooks'
-        branch: pre-commit-updates
+        commit-message: Update Python packages and pre-commit hooks
+        title: 'Update Python packages and pre-commit hooks'
+        branch: auto-package-updates
         base: master
         delete-branch: true
diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
@@ -12,23 +12,21 @@ concurrency:
 
 env:
   PYTHONWARNINGS: error
-
   BASE_IMAGE: openpilot-base
   CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_REGISTRY: ghcr.io/commaai
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
 
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  BUILD_CL: selfdrive/test/docker_build.sh cl
 
-  BUILD_CL: |
-      DOCKER_BUILDKIT=1 docker build --cache-to type=inline --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5
 
 jobs:
   build_release:
@@ -79,7 +77,9 @@ jobs:
     - uses: ./.github/workflows/setup
     - name: Build openpilot with all flags
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
-      run: ${{ env.RUN }} "scons -j$(nproc) --extras && release/check-dirty.sh"
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
     - name: Cleanup scons cache and rebuild
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache
       run: |
@@ -89,7 +89,7 @@ jobs:
       uses: actions/cache/save@v3
       if: github.ref == 'refs/heads/master'
       with:
-        path: ~/scons_cache
+        path: .ci_cache/scons_cache
         key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   build_mac:
@@ -192,23 +192,16 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
+    - name: Setup to push to repo
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
     - uses: ./.github/workflows/setup
       with:
         git-lfs: false
-    - name: Push to container registry
+    - name: Build and push CL Docker image
       run: |
-        $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
-        docker tag $DOCKER_REGISTRY/$BASE_IMAGE:latest $DOCKER_REGISTRY/$BASE_IMAGE:$GITHUB_SHA
-        docker push $DOCKER_REGISTRY/$BASE_IMAGE:$GITHUB_SHA
-    - name: Build CL Docker image
-      run: eval "$BUILD_CL"
-    - name: Push to container registry
-      run: |
-        $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest
-        docker tag $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest $DOCKER_REGISTRY/$CL_BASE_IMAGE:$GITHUB_SHA
-        docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:$GITHUB_SHA
+        eval "$BUILD_CL"
 
   static_analysis:
     name: static analysis
@@ -256,43 +249,26 @@ jobs:
       timeout-minutes: 40
       run: |
         ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
-                        $UNIT_TEST common && \
-                        $UNIT_TEST opendbc/can && \
-                        $UNIT_TEST selfdrive/boardd && \
-                        $UNIT_TEST selfdrive/controls && \
-                        $UNIT_TEST selfdrive/monitoring && \
-                        $UNIT_TEST system/loggerd && \
-                        $UNIT_TEST selfdrive/car && \
-                        $UNIT_TEST selfdrive/locationd && \
-                        $UNIT_TEST selfdrive/test/longitudinal_maneuvers && \
-                        $UNIT_TEST system/tests && \
-                        $UNIT_TEST system/ubloxd && \
+                        $PYTEST --rootdir . -c selfdrive/test/pytest-ci.ini && \
                         selfdrive/locationd/test/_test_locationd_lib.py && \
                         ./system/ubloxd/tests/test_glonass_runner && \
-                        $UNIT_TEST selfdrive/athena && \
-                        $UNIT_TEST selfdrive/thermald && \
-                        $UNIT_TEST system/hardware/tici && \
-                        $UNIT_TEST tools/lib/tests && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \
-                        ./common/tests/test_util && \
-                        ./common/tests/test_ratekeeper && \
-                        ./common/tests/test_swaglog && \
+                        ./common/tests/test_common && \
                         ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
                         ./system/loggerd/tests/test_logger &&\
                         ./system/proclogd/tests/test_proclog && \
                         ./tools/replay/tests/test_replay && \
                         ./tools/cabana/tests/test_cabana && \
                         ./system/camerad/test/ae_gray_test && \
-                        ./selfdrive/test/process_replay/test_fuzzy.py && \
-                        coverage xml"
+                        ./selfdrive/test/process_replay/test_fuzzy.py"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
   process_replay:
     name: process replay
-    runs-on: ubuntu-20.04
+    runs-on: ${{ ((github.event.pull_request.head.repo.full_name == 'commaai/openpilot') || (github.repository == 'commaai/openpilot')) && 'buildjet-8vcpu-ubuntu-2004' || 'ubuntu-20.04' }}
     steps:
     - uses: actions/checkout@v3
       with:
@@ -302,7 +278,7 @@ jobs:
       id: dependency-cache
       uses: actions/cache@v3
       with:
-        path: ~/comma_download_cache
+        path: .ci_cache/comma_download_cache
         key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
     - name: Build openpilot
       run: |
@@ -311,6 +287,7 @@ jobs:
       timeout-minutes: 30
       run: |
         ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache && \
                         coverage xml"
     - name: Print diff
       id: print-diff
@@ -325,7 +302,7 @@ jobs:
     - name: Upload reference logs
       if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
       run: |
-        ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
@@ -377,14 +354,14 @@ jobs:
       id: dependency-cache
       uses: actions/cache@v3
       with:
-        path: ~/comma_download_cache
+        path: .ci_cache/comma_download_cache
         key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
       timeout-minutes: 25
       run: |
-        ${{ env.RUN }} "pytest --cov --cov-report=xml -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
+        ${{ env.RUN }} "$PYTEST -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 5
@@ -409,9 +386,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
+          path: current
       - name: Save car docs diff
         id: save_diff
         run: |
+          cd current
           ${{ env.RUN }} "scons -j$(nproc)"
           output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
           output="${output//$'\n'/'%0A'}"

diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
@@ -1,10 +1,6 @@
 name: 'openpilot env setup'
 
 inputs:
-  setup_docker_scons_cache:
-    description: 'Whether or not to build the scons-cache docker image'
-    required: false
-    default: 'false'
   git_lfs:
     description: 'Whether or not to pull the git lfs'
     required: false
@@ -27,26 +23,29 @@ runs:
     - id: restore-scons-cache
       uses: actions/cache/restore@v3
       with:
-        path: ~/scons_cache
+        path: .ci_cache/scons_cache
         key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
           scons-${{ env.CACHE_COMMIT_DATE }}-
           scons-
+    # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      if: steps.restore-scons-cache.outputs.cache-hit != 'true'
+      run: mkdir -p $GITHUB_WORKSPACE/.ci_cache/scons_cache
     # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
     - id: normalize-file-permissions
       shell: bash
       name: Normalize file permissions to ensure a consistent docker build cache
       run: |
         find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
         find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    - id: setup-buildx-action
+      if: contains(runner.name, 'buildjet')
+      name: Set up Docker Buildx on buildjet to ensure a consistent cache
+      uses: docker/setup-buildx-action@v2
+      with:
+        driver: docker-container
     # build our docker image
     - shell: bash
-      run: eval ${{ env.BUILD }}
-    - id: setup-scons-cache-docker
-      name: Sets up a docker image with scons cache that can by mounted as a buildkit cache mount
-      shell: bash
-      if: ${{ inputs.setup_docker_scons_cache == 'true' }}
-      run: |
-        cp selfdrive/test/Dockerfile.scons_cache ~
-        cd ~
-        DOCKER_BUILDKIT=1 docker build -t scons-cache -f Dockerfile.scons_cache .
+      run: eval ${{ env.BUILD }}