Merge branch 'master' into PA-testing

Edison-CBS · Nov 4, 2023 · f23ad48 · f23ad48
2 parents 13e2a6e + bf932c6
commit f23ad48
Show file tree

Hide file tree

Showing 174 changed files with 3,001 additions and 4,567 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,5 +1,15 @@
 <!-- Please copy and paste the relevant template -->
 
+<!--- ***** Template: Fingerprint *****
+
+**Car**
+Which car (make, model, year) this fingerprint is for
+
+**Route**
+A route with the fingerprint
+
+-->
+
 <!--- ***** Template: Car bug fix *****
 
 **Description** [](A description of the bug and the fix. Also link any relevant issues.)

diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -0,0 +1,63 @@
+name: docs
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+concurrency:
+  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  BASE_IMAGE: openpilot-base
+
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+
+jobs:
+  docs:
+    name: build docs
+    runs-on: ubuntu-20.04
+    timeout-minutes: 45
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Build docs
+      run: |
+        ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make html"
+
+    - uses: actions/checkout@v4
+      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
+      with:
+        path: openpilot-docs
+        ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
+        repository: commaai/openpilot-docs
+    - name: Push
+      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
+      run: |
+        set -x
+
+        source release/identity.sh
+
+        cd openpilot-docs
+
+        git checkout --orphan tmp
+        git rm -rf --cached .
+
+        cp -r ../build/docs/html/ docs/
+        touch docs/.nojekyll
+        echo -n docs.comma.ai > docs/CNAME
+        git add -f .
+
+        git commit -m "build docs"
+
+        # docs live in different repo to not bloat openpilot's full clone size
+        git push -f origin gh-pages
diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml
@@ -9,7 +9,7 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: false
     - uses: actions/labeler@v5.0.0-alpha.1

diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml
@@ -24,7 +24,7 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
         check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - run: git lfs pull

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -27,7 +27,7 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build master-ci'
         check-regexp: ^((?!.*(build prebuilt).*).)*$
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0

diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml
@@ -12,7 +12,7 @@ jobs:
     container:
       image: ghcr.io/commaai/openpilot-base:latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: poetry lock
       run: |
         pip install poetry

diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - 'testing-closet*'
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
@@ -35,7 +36,7 @@ jobs:
     env:
       STRIPPED_DIR: /tmp/releasepilot
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - run: git lfs pull
@@ -77,7 +78,7 @@ jobs:
                (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
     runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -97,7 +98,7 @@ jobs:
     runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Setup to push to repo
@@ -120,7 +121,7 @@ jobs:
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
     needs: [docker_push]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: false
     - name: Setup docker
@@ -135,7 +136,7 @@ jobs:
     name: static analysis
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -147,7 +148,7 @@ jobs:
     name: valgrind
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -165,7 +166,7 @@ jobs:
     name: unit tests
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -190,7 +191,7 @@ jobs:
                    ((github.event_name != 'pull_request') || 
                     (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'buildjet-8vcpu-ubuntu-2004' || 'ubuntu-20.04' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -232,7 +233,7 @@ jobs:
     name: regen
     runs-on: 'ubuntu-20.04'
     steps: 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -259,7 +260,7 @@ jobs:
     name: model tests
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -295,7 +296,7 @@ jobs:
       matrix:
         job: [0, 1, 2, 3, 4]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -323,7 +324,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
           ref: ${{ github.event.pull_request.base.ref }}
@@ -333,7 +334,7 @@ jobs:
         run: |
           ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
           sudo chown -R $USER:$USER ${{ github.workspace }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
           path: current

diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 45
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -48,7 +48,7 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     timeout-minutes: 45
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
@@ -63,29 +63,11 @@ jobs:
       run: |
         selfdrive/test/docker_build.sh sim
 
-  docs:
-    name: build docs
-    runs-on: ubuntu-20.04
-    timeout-minutes: 45
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Setup to push to repo
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-      run: |
-        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
-        $DOCKER_LOGIN
-    - name: Build and push docs image
-      run: |
-        selfdrive/test/docker_build.sh docs
-
   devcontainer:
     name: devcontainer
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry

diff --git a/.gitignore b/.gitignore
@@ -79,6 +79,7 @@ comma*.sh
 
 selfdrive/modeld/thneed/compile
 selfdrive/modeld/models/*.thneed
+selfdrive/modeld/models/*.pkl
 
 *.bz2
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -41,7 +41,7 @@ repos:
         args: ['--explicit-package-bases']
         exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.292
+    rev: v0.1.3
     hooks:
     -   id: ruff
         exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -21,6 +21,8 @@ source ~/.bash_profile
 if [ -f /TICI ]; then
   source /etc/profile
 
+  rm -rf ~/.commacache
+
   if ! systemctl is-active --quiet systemd-resolved; then
     echo "restarting resolved"
     sudo systemctl start systemd-resolved
@@ -62,7 +64,9 @@ def deviceStage(String stageName, String deviceType, List env, def steps) {
     docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
       lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1) {
         timeout(time: 20, unit: 'MINUTES') {
-          device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          retry (3) {
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          }
           steps.each { item ->
             device(device_ip, item[0], item[1])
           }
@@ -154,7 +158,7 @@ node {
           ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR ./build_devel.sh"],
           ["build openpilot", "cd selfdrive/manager && ./build.py"],
           ["check dirty", "release/check-dirty.sh"],
-          ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
+          ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
           ["time to onroad", "cd selfdrive/test/ && pytest test_time_to_onroad.py"],
         ])
       },

diff --git a/README.md b/README.md
@@ -109,7 +109,6 @@ Directory Structure
     ├── third_party         # External libraries
     └── system              # Generic services
         ├── camerad         # Driver to capture images from the camera sensors
-        ├── clocksd         # Broadcasts current time
         ├── hardware        # Hardware abstraction classes
         ├── logcatd         # systemd journal as a service
         ├── loggerd         # Logger and uploader of car data

diff --git a/RELEASES.md b/RELEASES.md
@@ -2,6 +2,7 @@ Version 0.9.5 (2023-XX-XX)
 ========================
 * New driving model
   * Improved navigate on openpilot performance using navigation instructions as an additional model input
+* Cadillac Escalade ESV 2019 support thanks to twilsonco!
 * Hyundai Azera 2022 support thanks to sunnyhaibin!
 * Hyundai Azera Hybrid 2020 support thanks to chanhojung and haram-KONA!
 * Hyundai Custin 2023 support thanks to sunnyhaibin and Saber422!

diff --git a/SConstruct b/SConstruct
@@ -26,6 +26,10 @@ AddOption('--ubsan',
           action='store_true',
           help='turn on UBSan')
 
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
 AddOption('--clazy',
           action='store_true',
           help='build with clazy')
@@ -382,7 +386,6 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
-  'system/clocksd/SConscript',
   'system/proclogd/SConscript',
   'system/ubloxd/SConscript',
   'system/loggerd/SConscript',

diff --git a/cereal b/cereal
diff --git a/common/prefix.py b/common/prefix.py
@@ -14,6 +14,7 @@ def __init__(self, prefix: Optional[str] = None, clean_dirs_on_exit: bool = True
     self.clean_dirs_on_exit = clean_dirs_on_exit
 
   def __enter__(self):
+    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
     os.environ['OPENPILOT_PREFIX'] = self.prefix
     try:
       os.mkdir(self.msgq_path)
@@ -28,6 +29,8 @@ def __exit__(self, exc_type, exc_obj, exc_tb):
       self.clean_dirs()
     try:
       del os.environ['OPENPILOT_PREFIX']
+      if self.original_prefix is not None:
+        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
     except KeyError:
       pass
     return False