Merge branch 'master' into PA-dev2

Edison-CBS · Sep 8, 2024 · c133445 · c133445
2 parents f46bca0 + b286a2f
commit c133445
Show file tree

Hide file tree

Showing 105 changed files with 3,878 additions and 1,905 deletions.
diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
@@ -337,7 +337,6 @@ jobs:
     # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
     name: Create UI Report
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -353,5 +352,5 @@ jobs:
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:
-          name: report-${{ github.event.number }}
+          name: report-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
           path: selfdrive/ui/tests/test_ui/report_1/screenshots
diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml
@@ -43,37 +43,6 @@ jobs:
                         source selfdrive/test/setup_vsound.sh && \
                         CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
 
-  test_compatibility:
-    name: test 20.04 + 3.11
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Installing ubuntu dependencies
-      run: INSTALL_EXTRA_PACKAGES=no tools/install_ubuntu_dependencies.sh
-    - name: Installing python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11.4'
-    - name: Installing pip
-      run: pip install pip==24.0
-    - name: Installing uv
-      run: pip install uv
-    - name: git LFS
-      run: git lfs pull
-    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
-    - name: Getting scons cache
-      uses: 'actions/cache@v4'
-      with:
-        path: /tmp/scons_cache
-        key: scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-        restore-keys: |
-          scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}
-          scons-${{ runner.arch }}-ubuntu2004
-    - name: Building openpilot
-      run: uv run scons -u -j$(nproc)
-
   devcontainer:
     name: devcontainer
     runs-on: ubuntu-latest

diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml
@@ -1,5 +1,8 @@
 name: "ui preview"
 on:
+  push:
+    branches:
+      - master
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
     branches:
@@ -10,99 +13,151 @@ on:
 
 env:
   UI_JOB_NAME: "Create UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
 
 jobs:
   preview:
     if: github.repository == 'commaai/openpilot'
     name: preview
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 20
     permissions:
       contents: read
       pull-requests: write
       actions: read
     steps:
-      - name: Waiting for ui test to start
+      - name: Waiting for ui generation to start
         run: sleep 30
 
-      - name: Wait for ui report
+      - name: Waiting for ui generation to end
         uses: lewagon/wait-on-check-action@v1.3.4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ env.SHA }}
           check-name: ${{ env.UI_JOB_NAME }}
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allowed-conclusions: success
           wait-interval: 20
 
-      - name: Get workflow run ID
+      - name: Getting workflow run ID
         id: get_run_id
         run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ github.event.pull_request.head.sha }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
-      - name: Checkout ci-artifacts
-        uses: actions/checkout@v4
-        with:
-          repository: commaai/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/ci-artifacts
-          ref: master
-
-      - name: Download artifact
+      - name: Getting proposed ui
         id: download-artifact
         uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: report-${{ github.event.number }}
-          path: ${{ github.workspace }}/ci-artifacts
+          name: report-${{ env.REPORT_NAME }}
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui
+        uses: actions/checkout@v4
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui
+          ref: openpilot_master_ui
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui
+        run: |
+          git checkout --orphan=new_master_ui
+          git rm -rf *
+          git branch -D openpilot_master_ui
+          git branch -m openpilot_master_ui
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui --force
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: >-
+          sudo apt-get install -y imagemagick
+
+          scenes="homescreen settings_device offroad_alert update_available prime onroad onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body"
+          A=($scenes)
+
+          DIFF=""
+          TABLE="<details>"
+          TABLE="${TABLE}<summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
+
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+
+            if ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
+              convert -delay 20 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            else
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
+            fi
+          done
+
+          TABLE="${TABLE}</table>"
+          TABLE="${TABLE}</details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
 
-      - name: Push Screenshots
-        working-directory: ${{ github.workspace }}/ci-artifacts
+      - name: Saving proposed ui
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          git checkout -b openpilot/pr-${{ github.event.number }}
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
-          git add ${{ github.workspace }}/ci-artifacts/*
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
           git commit -m "screenshots for PR #${{ github.event.number }}"
-          git push origin openpilot/pr-${{ github.event.number }} --force
+          git push origin ${{ env.BRANCH_NAME }} --force
 
       - name: Comment Screenshots on PR
+        if: github.event_name == 'pull_request_target'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
             <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
-            ## UI Screenshots
-            <table>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/homescreen.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/settings_device.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/offroad_alert.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/update_available.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_sidebar.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_wide.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_wide_sidebar.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_alert_small.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_alert_mid.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_alert_full.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/driver_camera.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/body.png"></td>
-                <td></td>
-              </tr>
-            </table>
+            ## UI Preview
+            ${{ steps.find_diff.outputs.DIFF }}
           comment_tag: run_id_screenshots
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.lfsconfig b/.lfsconfig
@@ -0,0 +1,4 @@
+[lfs]
+	url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+	pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
+	locksverify = false
diff --git a/SConstruct b/SConstruct
@@ -67,7 +67,7 @@ AddOption('--pc-thneed',
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.islink(Dir('#rednose/').abspath), # minimal by default on release branch (where rednose is not a link)
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
 ## Architecture name breakdown (arch)

diff --git a/cereal/car.capnp b/cereal/car.capnp
@@ -87,7 +87,6 @@ struct OnroadEvent @0x9b1657f34caf3ad3 {
     startupNoCar @76;
     startupNoControl @77;
     startupMaster @78;
-    startupNoFw @104;
     fcw @79;
     steerSaturated @80;
     belowEngageSpeed @84;
@@ -100,16 +99,12 @@ struct OnroadEvent @0x9b1657f34caf3ad3 {
     cameraFrameRate @110;
     processNotRunning @95;
     dashcamMode @96;
-    controlsInitializing @98;
+    selfdriveInitializing @98;
     usbError @99;
-    roadCameraError @100;
-    driverCameraError @101;
-    wideRoadCameraError @102;
-    highCpuUsage @105;
     cruiseMismatch @106;
     lkasDisabled @107;
     canBusMissing @111;
-    controlsdLagging @112;
+    selfdrivedLagging @112;
     resumeBlocked @113;
     steerTimeLimit @115;
     vehicleSensorsInvalid @116;
@@ -147,6 +142,11 @@ struct OnroadEvent @0x9b1657f34caf3ad3 {
     brakeUnavailableDEPRECATED @2;
     plannerErrorDEPRECATED @32;
     gpsMalfunctionDEPRECATED @94;
+    roadCameraErrorDEPRECATED @100;
+    driverCameraErrorDEPRECATED @101;
+    wideRoadCameraErrorDEPRECATED @102;
+    highCpuUsageDEPRECATED @105;
+    startupNoFwDEPRECATED @104;
   }
 }
 
@@ -162,10 +162,13 @@ struct CarState {
   canErrorCounter @48 :UInt32;
 
   # car speed
-  vEgo @1 :Float32;          # best estimate of speed
-  aEgo @16 :Float32;         # best estimate of acceleration
-  vEgoRaw @17 :Float32;      # unfiltered speed from CAN sensors
-  vEgoCluster @44 :Float32;  # best estimate of speed shown on car's instrument cluster, used for UI
+  vEgo @1 :Float32;            # best estimate of speed
+  aEgo @16 :Float32;           # best estimate of aCAN cceleration
+  vEgoRaw @17 :Float32;        # unfiltered speed from wheel speed sensors
+  vEgoCluster @44 :Float32;    # best estimate of speed shown on car's instrument cluster, used for UI
+
+  vCruise @53 :Float32;        # actual set speed
+  vCruiseCluster @54 :Float32; # set speed to display in the UI
 
   yawRate @22 :Float32;     # best estimate of yaw rate
   standstill @18 :Bool;
@@ -505,6 +508,7 @@ struct CarParams {
   carFw @44 :List(CarFw);
 
   radarTimeStep @45: Float32 = 0.05;  # time delta between radar updates, 20Hz is very standard
+  radarDelay @74 :Float32;
   fingerprintSource @49: FingerprintSource;
   networkLocation @50 :NetworkLocation;  # Where Panda/C2 is integrated into the car's CAN network