Merge branch 'master' into PA-testing

.github/workflows/auto-cache-restore/action.yaml → .github/workflows/auto-cache/action.yaml

@@ -10,6 +10,10 @@ inputs:
   restore-keys:
     description: 'restore-keys'
     required: true
+  save:
+    description: 'whether to save the cache'
+    default: 'false'
+    required: false
 
 runs:
   using: "composite"
@@ -21,8 +25,16 @@ runs:
         path: ${{ inputs.path }}
 
     - name: setup github cache
-      if: ${{ !contains(runner.name, 'nsc') }}
-      uses: actions/cache/restore@v3
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
+      uses: 'actions/cache@v3'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: setup github cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
+      uses: 'actions/cache/restore@v3'
       with:
         path: ${{ inputs.path }}
         key: ${{ inputs.key }}
@@ -31,4 +43,7 @@ runs:
     # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
     - id: scons-cache-setup
       shell: bash
-      run: mkdir -p ${{ inputs.path }}
+      run: |
+        mkdir -p ${{ inputs.path }}
+        sudo chmod -R 777 ${{ inputs.path }}
+        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/selfdrive_tests.yaml

@@ -20,7 +20,7 @@ env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   BUILD_CL: selfdrive/test/docker_build.sh cl
 
@@ -42,6 +42,7 @@ jobs:
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
+    - uses: ./.github/workflows/setup-pre-commit
     - uses: ./.github/workflows/setup-with-retry
     - name: Check submodules
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
@@ -66,7 +67,7 @@ jobs:
         cp pyproject.toml $STRIPPED_DIR
         cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
   build:
     strategy:
@@ -137,10 +138,11 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: true
+    - uses: ./.github/workflows/setup-pre-commit
     - uses: ./.github/workflows/setup-with-retry
     - name: pre-commit
       timeout-minutes: 4
-      run: ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all"
+      run: ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files && pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
   valgrind:
     name: valgrind
@@ -180,7 +182,7 @@ jobs:
       run: |
         ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                         export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
-                        $PYTEST --timeout 40 -m 'not slow' -n $(nproc) && \
+                        $PYTEST --timeout 60 -m 'not slow' -n $(nproc) && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py"
@@ -305,7 +307,9 @@ jobs:
 
   test_cars:
     name: cars
-    runs-on: ubuntu-20.04
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') || 
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
     strategy:
       fail-fast: false
       matrix:

.github/workflows/setup-pre-commit/action.yaml

@@ -0,0 +1,12 @@
+name: 'set up pre-commit environment'
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/workflows/auto-cache
+      with: 
+        path: .ci_cache/pre-commit
+        key: pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
+        restore-keys: |
+          pre-commit-
+        save: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' }}

.github/workflows/setup/action.yaml

@@ -44,7 +44,7 @@ runs:
     - shell: bash
       run: echo "$CACHE_COMMIT_DATE"
     - id: scons-cache
-      uses: ./.github/workflows/auto-cache-restore
+      uses: ./.github/workflows/auto-cache
       with:
         path: .ci_cache/scons_cache
         key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

Jenkinsfile

@@ -168,7 +168,7 @@ node {
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
   def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'dashcam3', 'dashcam3-staging', 'testing-closet*', 'hotfix-*']
+                         'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
   if (env.BRANCH_NAME != 'master') {
@@ -180,7 +180,7 @@ node {
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release3-staging", "tici-needs-can", [], [
-        ["build release3-staging & dashcam3-staging", "RELEASE_BRANCH=release3-staging DASHCAM_BRANCH=dashcam3-staging $SOURCE_DIR/release/build_release.sh"],
+        ["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"],
       ])
     }
 

SConstruct

@@ -297,8 +297,11 @@ else:
   qt_env['QTDIR'] = qt_install_prefix
   qt_dirs = [
     f"{qt_install_headers}",
-    f"{qt_install_headers}/QtGui/5.12.8/QtGui",
   ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
   qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
 
   qt_libs = [f"Qt5{m}" for m in qt_modules]

common/params.cc

@@ -184,7 +184,6 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"PandaSignatures", CLEAR_ON_MANAGER_START},
-    {"Passive", PERSISTENT},
     {"PrimeType", PERSISTENT},
     {"RecordFront", PERSISTENT},
     {"RecordFrontLock", PERSISTENT},  // for the internal fleet

common/swaglog.cc

@@ -10,6 +10,7 @@
 #include <string>
 
 #include <zmq.h>
+#include <stdarg.h>
 #include "third_party/json11/json11.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"

launch_env.sh

@@ -10,8 +10,4 @@ if [ -z "$AGNOS_VERSION" ]; then
   export AGNOS_VERSION="9.1"
 fi
 
-if [ -z "$PASSIVE" ]; then
-  export PASSIVE="1"
-fi
-
 export STAGING_ROOT="/data/safe_staging"

launch_openpilot.sh

@@ -1,5 +1,3 @@
 #!/usr/bin/bash
 
-export PASSIVE="0"
 exec ./launch_chffrplus.sh
-

release/build_release.sh

@@ -102,11 +102,4 @@ if [ ! -z "$RELEASE_BRANCH" ]; then
   git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH
 fi
 
-if [ ! -z "$DASHCAM_BRANCH" ]; then
-  # Create dashcam
-  git rm selfdrive/car/*/carcontroller.py
-  git commit -m "create dashcam release from release"
-  git push -f origin $RELEASE_BRANCH:$DASHCAM_BRANCH
-fi
-
 echo "[-] done T=$SECONDS"

release/verify.sh

@@ -6,7 +6,7 @@ RED="\033[0;31m"
 GREEN="\033[0;32m"
 CLEAR="\033[0m"
 
-BRANCHES="devel dashcam3 release3"
+BRANCHES="devel release3"
 for b in $BRANCHES; do
   if git diff --quiet origin/$b origin/$b-staging && [ "$(git rev-parse origin/$b)" = "$(git rev-parse origin/$b-staging)" ]; then
     printf "%-10s $GREEN ok $CLEAR\n" "$b"

scripts/build_small.sh

@@ -26,7 +26,7 @@ cd $OUT
 git tag -l | xargs git tag -d
 
 # remove non-master branches
-BRANCHES="release2 dashcam dashcam3 release3 devel master-ci nightly"
+BRANCHES="release2 release3 devel master-ci nightly"
 for branch in $BRANCHES; do
   git branch -D $branch
   git branch -D ${branch}-staging || true

selfdrive/controls/controlsd.py

@@ -106,14 +106,13 @@ def __init__(self, CI=None):
     self.is_metric = self.params.get_bool("IsMetric")
     self.is_ldw_enabled = self.params.get_bool("IsLdwEnabled")
     openpilot_enabled_toggle = self.params.get_bool("OpenpilotEnabledToggle")
-    passive = self.params.get_bool("Passive") or not openpilot_enabled_toggle
 
     # detect sound card presence and ensure successful init
     sounds_available = HARDWARE.get_sound_card_online()
 
     car_recognized = self.CP.carName != 'mock'
 
-    controller_available = self.CI.CC is not None and not passive and not self.CP.dashcamOnly
+    controller_available = self.CI.CC is not None and openpilot_enabled_toggle and not self.CP.dashcamOnly
     self.CP.passive = not car_recognized or not controller_available or self.CP.dashcamOnly
     if self.CP.passive:
       safety_config = car.CarParams.SafetyConfig.new_message()

selfdrive/controls/tests/test_startup.py

@@ -69,7 +69,6 @@ def test_startup_alert(self, expected_event, car_model, fw_versions, brand):
     pm = messaging.PubMaster(['can', 'pandaStates'])
 
     params = Params()
-    params.put_bool("Passive", False)
     params.put_bool("OpenpilotEnabledToggle", True)
 
     # Build capnn version of FW array

selfdrive/manager/manager.py

@@ -60,13 +60,6 @@ def manager_init() -> None:
     if params.get(k) is None:
       params.put(k, v)
 
-  # is this dashcam?
-  if os.getenv("PASSIVE") is not None:
-    params.put_bool("Passive", bool(int(os.getenv("PASSIVE", "0"))))
-
-  if params.get("Passive") is None:
-    raise Exception("Passive must be set to continue")
-
   # Create folders needed for msgq
   try:
     os.mkdir("/dev/shm")

selfdrive/manager/test/test_manager.py

@@ -21,7 +21,6 @@
 @pytest.mark.tici
 class TestManager(unittest.TestCase):
   def setUp(self):
-    os.environ['PASSIVE'] = '0'
     HARDWARE.set_power_save(False)
 
     # ensure clean CarParams
@@ -53,7 +52,6 @@ def test_clean_exit(self):
     """
     HARDWARE.set_power_save(False)
     manager.manager_init()
-    manager.manager_prepare()
 
     CP = car.CarParams.new_message()
     procs = ensure_running(managed_processes.values(), True, Params(), CP, not_run=BLACKLIST_PROCS)

selfdrive/test/helpers.py

@@ -12,7 +12,6 @@
 
 
 def set_params_enabled():
-  os.environ['PASSIVE'] = "0"
   os.environ['REPLAY'] = "1"
   os.environ['FINGERPRINT'] = "TOYOTA COROLLA TSS2 2019"
   os.environ['LOGPRINT'] = "debug"
@@ -21,7 +20,6 @@ def set_params_enabled():
   params.put("HasAcceptedTerms", terms_version)
   params.put("CompletedTrainingVersion", training_version)
   params.put_bool("OpenpilotEnabledToggle", True)
-  params.put_bool("Passive", False)
 
   # valid calib
   msg = messaging.new_message('liveCalibration')

selfdrive/test/process_replay/process_replay.py

@@ -725,7 +725,6 @@ def _replay_multi_process(
 def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=None) -> Dict[str, Any]:
   params_dict = {
     "OpenpilotEnabledToggle": True,
-    "Passive": False,
     "DisengageOnAccelerator": True,
     "DisableLogging": False,
   }

selfdrive/thermald/thermald.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import datetime
 import os
 import json
 import queue
@@ -15,7 +14,6 @@
 from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.common.dict_helpers import strip_deprecated_keys
-from openpilot.common.time import MIN_DATE
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.params import Params
 from openpilot.common.realtime import DT_TRML
@@ -293,19 +291,13 @@ def thermald_thread(end_event, hw_queue) -> None:
 
     # **** starting logic ****
 
-    # Ensure date/time are valid
-    now = datetime.datetime.utcnow()
-    startup_conditions["time_valid"] = now > MIN_DATE
-    set_offroad_alert_if_changed("Offroad_InvalidTime", (not startup_conditions["time_valid"]) and peripheral_panda_present)
-
     startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate")
     startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall")
     startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version
 
     # with 2% left, we killall, otherwise the phone will take a long time to boot
     startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2
-    startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version or \
-                                               params.get_bool("Passive")
+    startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version
     startup_conditions["not_driver_view"] = not params.get_bool("IsDriverViewEnabled")
     startup_conditions["not_taking_snapshot"] = not params.get_bool("IsTakingSnapshot")
 

selfdrive/ui/SConscript

@@ -19,6 +19,9 @@ if arch == "Darwin":
   del base_libs[base_libs.index('OpenCL')]
   qt_env['FRAMEWORKS'] += ['OpenCL']
 
+# FIXME: remove this once we're on 5.15 (24.04)
+qt_env['CXXFLAGS'] += ["-Wno-deprecated-declarations"]
+
 qt_util = qt_env.Library("qt_util", ["#selfdrive/ui/qt/api.cc", "#selfdrive/ui/qt/util.cc"], LIBS=base_libs)
 widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc",
                "qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc",
@@ -98,23 +101,17 @@ if GetOption('extras') and arch != "Darwin":
   senv['LINKFLAGS'].append('-Wl,-strip-debug')
 
   release = "release3"
-  dashcam = "dashcam3"
   installers = [
     ("openpilot", release),
     ("openpilot_test", f"{release}-staging"),
     ("openpilot_nightly", "nightly"),
     ("openpilot_internal", "master"),
-    ("dashcam", dashcam),
-    ("dashcam_test", f"{dashcam}-staging"),
   ]
 
-  cont = {}
-  for brand in ("openpilot", "dashcam"):
-    cont[brand] = senv.Command(f"installer/continue_{brand}.o", f"installer/continue_{brand}.sh",
-                               "ld -r -b binary -o $TARGET $SOURCE")
+  cont = senv.Command(f"installer/continue_openpilot.o", f"installer/continue_openpilot.sh",
+                      "ld -r -b binary -o $TARGET $SOURCE")
   for name, branch in installers:
-    brand = "dashcam" if "dashcam" in branch else "openpilot"
-    d = {'BRANCH': f"'\"{branch}\"'", 'BRAND': f"'\"{brand}\"'"}
+    d = {'BRANCH': f"'\"{branch}\"'"}
     if "internal" in name:
       d['INTERNAL'] = "1"
 
@@ -123,7 +120,7 @@ if GetOption('extras') and arch != "Darwin":
       r.raise_for_status()
       d['SSH_KEYS'] = f'\\"{r.text.strip()}\\"'
     obj = senv.Object(f"installer/installers/installer_{name}.o", ["installer/installer.cc"], CPPDEFINES=d)
-    f = senv.Program(f"installer/installers/installer_{name}", [obj, cont[brand]], LIBS=qt_libs)
+    f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=qt_libs)
     # keep installers small
     assert f[0].get_size() < 350*1e3
 

selfdrive/ui/installer/installer.cc

@@ -33,8 +33,8 @@ const QString CACHE_PATH = "/data/openpilot.cache";
 #define INSTALL_PATH "/data/openpilot"
 #define TMP_INSTALL_PATH "/data/tmppilot"
 
-extern const uint8_t str_continue[] asm("_binary_selfdrive_ui_installer_continue_" BRAND "_sh_start");
-extern const uint8_t str_continue_end[] asm("_binary_selfdrive_ui_installer_continue_" BRAND "_sh_end");
+extern const uint8_t str_continue[] asm("_binary_selfdrive_ui_installer_continue_openpilot_sh_start");
+extern const uint8_t str_continue_end[] asm("_binary_selfdrive_ui_installer_continue_openpilot_sh_end");
 
 bool time_valid() {
   time_t rawtime;

selfdrive/ui/qt/offroad/onboarding.cc

@@ -183,7 +183,7 @@ void DeclinePage::showEvent(QShowEvent *event) {
 void OnboardingWindow::updateActiveScreen() {
   if (!accepted_terms) {
     setCurrentIndex(0);
-  } else if (!training_done && !params.getBool("Passive")) {
+  } else if (!training_done) {
     setCurrentIndex(1);
   } else {
     emit onboardingDone();