Merge branch 'commaai:master' into PA-testing

.github/workflows/ci_weekly_report.yaml

@@ -0,0 +1,123 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    if: github.repository == 'commaai/openpilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.value }}
+    steps:
+      - id: ci_runs_setup
+        run: |
+          CI_RUNS=${{ inputs.ci_runs || '50' }}
+          mylist="value=["
+
+          for i in $(seq 1 $CI_RUNS);
+          do
+              if [ $i != $CI_RUNS ]; then
+                  mylist+="\"$i\", "
+              else
+                  mylist+="\"$i\"]"
+              fi
+          done
+
+          echo "$mylist" >> $GITHUB_OUTPUT
+          echo "Number of CI runs for report: $CI_RUNS"
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix:
+        value: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.value }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              const jobName = job.name.split('/')[2].trim();
+              report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push(job.html_url); break;
+                case "failure":
+                  report[jobName].failures.push(job.html_url); break;
+                case "cancelled":
+                  report[jobName].cancelled.push(job.html_url); break;
+              }
+            });
+            return JSON.stringify(report);
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          echo $JOB_RESULTS > job_results.json
+          generate_html_table() {
+            echo "<table>"
+            echo "<thead>"
+            echo "  <tr>"
+            echo "    <th>Job</th>"
+            echo "    <th>Succeeded ✅</th>"
+            echo "    <th>Failed ❌</th>"
+            echo "    <th>Cancelled (timed out) ⏰</th>"
+            echo "  </tr>"
+            echo "</thead>"
+            jq -r '
+              "<tbody>",
+              keys[] as $job |
+              "<tr>",
+              "  <td>\($job)</td>",
+              "  <td>",
+              "    <details>",
+              "      <summary>(\(.[$job].successes | length))</summary>",
+              "      \(.[$job].successes[])<br>",
+              "    </details>",
+              "  </td>",
+              "  <td>",
+              "    <details>",
+              "      <summary>(\(.[$job].failures | length))</summary>",
+              "      \(.[$job].failures[])<br>",
+              "    </details>",
+              "  </td>",
+              "  <td>",
+              "    <details>",
+              "      <summary>(\(.[$job].cancelled | length))</summary>",
+              "      \(.[$job].cancelled[])<br>",
+              "    </details>",
+              "  </td>",
+              "</tr>"
+            ' job_results.json
+            echo  "</tbody>"
+            echo  "</table>"
+          }
+          echo "# CI Job Summary" >> $GITHUB_STEP_SUMMARY
+          generate_html_table >> $GITHUB_STEP_SUMMARY
+

.github/workflows/ci_weekly_run.yaml

@@ -0,0 +1,21 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  selfdrive_tests:
+    uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}
+  tools_tests:
+    uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}

.github/workflows/docs.yaml

@@ -5,9 +5,14 @@ on:
     branches:
       - master
   pull_request:
-
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:

.github/workflows/repo-maintenance.yaml

@@ -43,8 +43,9 @@ jobs:
     - uses: actions/checkout@v4
     - name: uv lock
       run: |
-        pip install uv
-        uv lock
+        python3 -m ensurepip --upgrade
+        pip3 install uv
+        uv lock --upgrade
     - name: pre-commit autoupdate
       run: |
         git config --global --add safe.directory '*'

.github/workflows/selfdrive_tests.yaml

@@ -6,9 +6,15 @@ on:
       - 'testing-closet*'
   pull_request:
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:
@@ -320,5 +326,5 @@ jobs:
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:
-          name: report
-          path: selfdrive/ui/tests/test_ui/report
+          name: report-${{ inputs.run_number }}
+          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}

.github/workflows/tools_tests.yaml

@@ -5,9 +5,14 @@ on:
     branches-ignore:
       - 'testing-closet*'
   pull_request:
-
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:

cereal/messaging/tests/test_services.py

@@ -15,6 +15,7 @@ class TestServices(unittest.TestCase):
   def test_services(self, s):
     service = SERVICE_LIST[s]
     self.assertTrue(service.frequency <= 104)
+    self.assertTrue(service.decimation != 0)
 
   def test_generated_header(self):
     with tempfile.NamedTemporaryFile(suffix=".h") as f:

cereal/services.py

@@ -61,7 +61,7 @@ def __init__(self, should_log: bool, frequency: float, decimation: Optional[int]
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., 0),
+  "modelV2": (True, 20.),
   "managerState": (True, 2., 1),
   "uploaderState": (True, 0., 1),
   "navInstruction": (True, 1., 10),

selfdrive/car/ford/carstate.py

@@ -15,7 +15,7 @@ def __init__(self, CP):
     super().__init__(CP)
     can_define = CANDefine(DBC[CP.carFingerprint]["pt"])
     if CP.transmissionType == TransmissionType.automatic:
-      self.shifter_values = can_define.dv["Gear_Shift_by_Wire_FD1"]["TrnRng_D_RqGsm"]
+      self.shifter_values = can_define.dv["PowertrainData_10"]["TrnRng_D_Rq"]
 
     self.vehicle_sensors_valid = False
 
@@ -69,7 +69,7 @@ def update(self, cp, cp_cam):
 
     # gear
     if self.CP.transmissionType == TransmissionType.automatic:
-      gear = self.shifter_values.get(cp.vl["Gear_Shift_by_Wire_FD1"]["TrnRng_D_RqGsm"])
+      gear = self.shifter_values.get(cp.vl["PowertrainData_10"]["TrnRng_D_Rq"])
       ret.gearShifter = self.parse_gear_shifter(gear)
     elif self.CP.transmissionType == TransmissionType.manual:
       ret.clutchPressed = cp.vl["Engine_Clutch_Data"]["CluPdlPos_Pc_Meas"] > 0
@@ -139,7 +139,7 @@ def get_can_parser(CP):
 
     if CP.transmissionType == TransmissionType.automatic:
       messages += [
-        ("Gear_Shift_by_Wire_FD1", 10),
+        ("PowertrainData_10", 10),
       ]
     elif CP.transmissionType == TransmissionType.manual:
       messages += [

selfdrive/car/tests/test_car_interfaces.py

@@ -62,6 +62,7 @@ def test_car_interfaces(self, car_name, data):
 
     car_params = CarInterface.get_params(car_name, args['fingerprints'], args['car_fw'],
                                          experimental_long=args['experimental_long'], docs=False)
+    car_params = car_params.as_reader()
     car_interface = CarInterface(car_params, CarController, CarState)
     assert car_params
     assert car_interface

selfdrive/controls/controlsd.py

@@ -65,8 +65,7 @@ def __init__(self, CI=None):
     if CI is None:
       cloudlog.info("controlsd is waiting for CarParams")
       with car.CarParams.from_bytes(self.params.get("CarParams", block=True)) as msg:
-        # TODO: this shouldn't need to be a builder
-        self.CP = msg.as_builder()
+        self.CP = msg
       cloudlog.info("controlsd got CarParams")
 
       # Uses car interface helper functions, altering state won't be considered by card for actuation

selfdrive/controls/lib/latcontrol_torque.py

@@ -25,7 +25,7 @@
 class LatControlTorque(LatControl):
   def __init__(self, CP, CI):
     super().__init__(CP, CI)
-    self.torque_params = CP.lateralTuning.torque
+    self.torque_params = CP.lateralTuning.torque.as_builder()
     self.pid = PIDController(self.torque_params.kp, self.torque_params.ki,
                              k_f=self.torque_params.kf, pos_limit=self.steer_max, neg_limit=-self.steer_max)
     self.torque_from_lateral_accel = CI.torque_from_lateral_accel()

selfdrive/controls/lib/tests/test_latcontrol.py

@@ -21,7 +21,7 @@ def test_saturation(self, car_name, controller):
     CI = CarInterface(CP, CarController, CarState)
     VM = VehicleModel(CP)
 
-    controller = controller(CP, CI)
+    controller = controller(CP.as_reader(), CI)
 
     CS = car.CarState.new_message()
     CS.vEgo = 30

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-8737e368e17f859291164286feb4246e00c0b4a5
+f6ff3601bd0496e78d8bc3b019d58bb7739f096b

selfdrive/ui/tests/test_ui/run.py

@@ -116,7 +116,7 @@ def setup_onroad_sidebar(click, pm: PubMaster):
 
 TEST_DIR = pathlib.Path(__file__).parent
 
-TEST_OUTPUT_DIR = TEST_DIR / "report"
+TEST_OUTPUT_DIR = TEST_DIR / "report_1"
 SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots"