Merge remote-tracking branch 'commaai/panda/master' into camera-scc-long

# Conflicts: # board/safety/safety_hyundai.h
commaai · Aug 15, 2023 · a42c892 · a42c892
2 parents dfc38c1 + d7f75ca
commit a42c892
Show file tree

Hide file tree

Showing 201 changed files with 6,915 additions and 2,593 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -46,6 +46,10 @@ jobs:
         run: ${{ env.RUN }} "scons -j4"
       - name: Build panda with SPI support
         run: ${{ env.RUN }} "ENABLE_SPI=1 scons -j4"
+      - name: Build with UBSan
+        run: ${{ env.RUN }} "scons -j4 --ubsan"
+      - name: Build jungle firmware with FINAL_PROVISIONING support
+        run: ${{ env.RUN }} "FINAL_PROVISIONING=1 scons -j4 board/jungle"
 
   unit_tests:
     name: unit tests
@@ -59,38 +63,30 @@ jobs:
         run: $RUN "scons -j4"
       - name: Test communication protocols
         run: $RUN "cd tests/usbprotocol && ./test.sh"
+      - name: Test logging
+        run: $RUN "cd tests/logging && ./test.sh"
 
   safety:
     name: safety
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        flags: ['', '--ubsan']
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
       - name: Build Docker image
         run: eval "$BUILD"
       - name: Run safety tests
-        timeout-minutes: 3
+        timeout-minutes: 5
         run: |
           ${{ env.RUN }} "cd .. && \
                           scons -c && \
                           scons -j$(nproc) opendbc/ cereal/ && \
                           cd panda && \
-                          scons -j$(nproc) && \
+                          scons -j$(nproc) ${{ matrix.flags }} && \
                           tests/safety/test.sh"
 
-  safety_replay:
-    name: safety replay
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v2
-      - name: Build Docker image
-        run: eval "$BUILD"
-      - name: Run safety replay
-        run: ${{ env.RUN }} "cd tests/safety_replay &&
-                             scons -u .. &&
-                             ./test_safety_replay.py"
-
   misra:
     name: misra c2012
     runs-on: ubuntu-20.04

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+*.tmp
 *.pyc
 .*.swp
 .*.swo
@@ -11,7 +12,7 @@ a.out
 .#*
 dist/
 pandacan.egg-info/
-board/obj/
+obj/
 examples/output.csv
 .DS_Store
 .vscode*

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,36 +1,18 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
-    hooks:
-    -   id: check-ast
-    -   id: check-yaml
-    -   id: check-merge-conflict
-    -   id: check-symlinks
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
-    hooks:
-    -   id: mypy
-        additional_dependencies: ['git+https://github.com/numpy/numpy-stubs', 'types-requests', 'types-atomicwrites',
-                                  'types-pycurl']
--   repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-    -   id: flake8
-        args:
-        - --select=F,E112,E113,E304,E501,E502,E701,E702,E703,E71,E72,E731,W191,W6
-        - --exclude=tests/gmbitbang/*
-        - --max-line-length=160
-        - --statistics
--   repo: local
-    hooks:
-    -   id: pylint
-        name: pylint
-        entry: pylint
-        language: system
-        types: [python]
-        args:
-        - -rn
-        - -sn
-        - -j0
-        - --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,W1203,W1514,E1136
-        - --generated-members="usb1.*"
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+    - id: check-ast
+    - id: check-yaml
+    - id: check-merge-conflict
+    - id: check-symlinks
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.4.1
+  hooks:
+    - id: mypy
+      additional_dependencies: ['git+https://github.com/numpy/numpy-stubs', 'types-requests', 'types-atomicwrites',
+                                'types-pycurl']
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.0.282
+  hooks:
+    - id: ruff
diff --git a/Dockerfile b/Dockerfile
@@ -33,6 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     pkg-config \
     python \
     python-dev \
+    qt5-default \
     unzip \
     wget \
     zlib1g-dev \
@@ -50,12 +51,12 @@ RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-instal
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
 
 ENV PANDA_PATH=/tmp/openpilot/panda
-ENV OPENPILOT_REF="ee0dd36a3c775dbd82493c84f4e7272c1eb3fcbd"
-ENV OPENDBC_REF="342c0320dd271fb585db3cced397c5122078af85"
+ENV OPENPILOT_REF="80bbba14f74e57bbe90216dfd0a99f6f68d77ca2"
+ENV OPENDBC_REF="5880fbbccf5a670631b51836f20e446de643795a"
 
 COPY requirements.txt /tmp/
-RUN pyenv install 3.8.10 && \
-    pyenv global 3.8.10 && \
+RUN pyenv install 3.11.4 && \
+    pyenv global 3.11.4 && \
     pyenv rehash && \
     pip install --no-cache-dir -r /tmp/requirements.txt
 
@@ -69,23 +70,16 @@ RUN cd /tmp && \
     cd /tmp/tmppilot && \
     git fetch origin $OPENPILOT_REF && \
     git checkout $OPENPILOT_REF && \
-    git submodule update --init cereal opendbc rednose_repo && \
+    git submodule update --init cereal opendbc rednose_repo body && \
     git -C opendbc fetch && \
     git -C opendbc checkout $OPENDBC_REF && \
     git -C opendbc reset --hard HEAD && \
     git -C opendbc clean -xfd && \
     mkdir /tmp/openpilot && \
-    cp -pR SConstruct site_scons/ tools/ selfdrive/ system/ common/ cereal/ opendbc/ rednose/ third_party/ /tmp/openpilot && \
+    cp -pR SConstruct site_scons/ tools/ selfdrive/ system/ common/ cereal/ opendbc/ rednose/ third_party/ body/ /tmp/openpilot && \
     rm -rf /tmp/openpilot/panda && \
     rm -rf /tmp/tmppilot
 
-RUN cd /tmp/openpilot && \
-    git clone https://github.com/commaai/panda_jungle.git && \
-    cd panda_jungle && \
-    git fetch && \
-    git checkout 3a791be1f1877a69cf45de16a670992380622297 && \
-    rm -rf .git/
-
 RUN cd /tmp/openpilot && \
     pip install --no-cache-dir -r opendbc/requirements.txt && \
     pip install --no-cache-dir --upgrade aenum lru-dict pycurl tenacity atomicwrites serial smbus2

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,7 +1,6 @@
 def docker_run(String step_label, int timeout_mins, String cmd) {
   timeout(time: timeout_mins, unit: 'MINUTES') {
     sh script: "docker run --rm --privileged \
-          --env PARTIAL_TESTS=${env.PARTIAL_TESTS} \
           --env PYTHONWARNINGS=error \
           --volume /dev/bus/usb:/dev/bus/usb \
           --volume /var/run/dbus:/var/run/dbus \
@@ -62,7 +61,6 @@ pipeline {
   agent any
   environment {
     CI = "1"
-    PARTIAL_TESTS = "${env.BRANCH_NAME == 'master' ? ' ' : '1'}"
     PYTHONWARNINGS= "error"
     DOCKER_IMAGE_TAG = "panda:build-${env.GIT_COMMIT}"
 
@@ -82,7 +80,8 @@ pipeline {
           steps {
             phone_steps("panda-dos", [
               ["build", "scons -j4"],
-              ["flash", "cd tests/ && ./ci_reset_internal_hw.py"],
+              ["flash", "cd tests/ && ./reflash_internal_panda.py"],
+              ["flash jungle", "cd board/jungle && ./flash.py"],
               ["test", "cd tests/hitl && HW_TYPES=6 pytest --durations=0 [2-7]*.py -k 'not test_send_recv'"],
             ])
           }
@@ -93,8 +92,9 @@ pipeline {
           steps {
             phone_steps("panda-tres", [
               ["build", "scons -j4"],
-              ["flash", "cd tests/ && ./ci_reset_internal_hw.py"],
-              ["test", "cd tests/hitl && HW_TYPES=9 pytest --durations=0 2*.py [5-7]*.py -k 'not test_fan_controller and not test_fan_overshoot'"],
+              ["flash", "cd tests/ && ./reflash_internal_panda.py"],
+              ["flash jungle", "cd board/jungle && ./flash.py"],
+              ["test", "cd tests/hitl && HW_TYPES=9 pytest --durations=0 2*.py [5-9]*.py"],
             ])
           }
         }
@@ -117,7 +117,9 @@ pipeline {
             stage('prep') {
               steps {
                 script {
-                  docker_run("reset hardware", 3, "python ./tests/ci_reset_hw.py")
+                  retry (3) {
+                    docker_run("reset hardware", 3, "python ./tests/ci_reset_hw.py")
+                  }
                 }
               }
             }
@@ -131,7 +133,8 @@ pipeline {
             stage('HITL tests') {
               steps {
                 script {
-                  docker_run("HITL tests", 35, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/test.sh')
+                  docker_run("parallel tests", 5, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/run_parallel_tests.sh')
+                  docker_run("serial tests", 9, 'PANDAS_JUNGLE=23002d000851393038373731 PANDAS_EXCLUDE="1d0002000c51303136383232 2f002e000c51303136383232" ./tests/hitl/run_serial_tests.sh')
                 }
               }
             }

diff --git a/README.md b/README.md
@@ -20,8 +20,8 @@ sudo apt-get install dfu-util gcc-arm-none-eabi python3-pip libffi-dev git
 ```
 ```bash
 # macOS
-brew tap ArmMbed/homebrew-formulae
-brew install python dfu-util arm-none-eabi-gcc gcc@12
+brew install --cask gcc-arm-embedded
+brew install python3 dfu-util gcc@12
 ```
 
 Clone panda repository:
@@ -112,7 +112,7 @@ to ensure that the behavior remains unchanged.
     * compiling the code and flashing it through USB.
     * receiving, sending, and forwarding CAN messages on all buses, over USB.
 
-In addition, we run the [pylint](https://www.pylint.org/) and [flake8](https://github.com/PyCQA/flake8) linters on all python files within the panda repo.
+In addition, we run the [ruff linter](https://github.com/astral-sh/ruff) on all python files within the panda repo.
 
 ## Hardware