Merge branch 'master' into PA-dev

.github/labeler.yaml

@@ -4,59 +4,7 @@ CI / testing:
 
 car:
   - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/**'
-
-body:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/body/*'
-
-chrysler:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/chrysler/*'
-
-ford:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/ford/*'
-
-gm:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/gm/*'
-
-honda:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/honda/*'
-
-hyundai:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/hyundai/*'
-
-mazda:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/mazda/*'
-
-nissan:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/nissan/*'
-
-subaru:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/subaru/*'
-
-tesla:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/tesla/*'
-
-toyota:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/toyota/*'
-
-volkswagen:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/volkswagen/*'
-
-fingerprint:
-  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/*/fingerprints.py'
+    - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
 
 simulation:
   - changed-files:
@@ -74,6 +22,6 @@ multilanguage:
   - changed-files:
     - any-glob-to-all-files: 'selfdrive/ui/translations/**'
 
-research:
+autonomy:
   - changed-files:
     - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"

.github/workflows/repo-maintenance.yaml

@@ -26,6 +26,11 @@ jobs:
         git config --global --add safe.directory '*'
         git -c submodule."tinygrad".update=none submodule update --remote
         git add .
+    - name: update car docs
+      run: |
+        scons -j$(nproc) --minimal opendbc
+        PYTHONPATH=. python selfdrive/car/docs.py
+        git add docs/CARS.md
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:

SConstruct

@@ -168,9 +168,6 @@ else:
 if arch != "Darwin":
   ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
 
-# Enable swaglog include in submodules
-cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
-
 ccflags_option = GetOption('ccflags')
 if ccflags_option:
   ccflags += ccflags_option.split(' ')
@@ -185,12 +182,9 @@ env = Environment(
     "-Werror",
     "-Wshadow",
     "-Wno-unknown-warning-option",
-    "-Wno-deprecated-register",
-    "-Wno-register",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
-    "-Wno-error=unused-but-set-variable",
     "-Wno-vla-cxx-extension",
   ] + cflags + ccflags,
 
@@ -207,7 +201,6 @@ env = Environment(
     "#third_party",
     "#cereal",
     "#msgq",
-    "#opendbc/can",
   ],
 
   CC='clang',
@@ -273,7 +266,7 @@ Export('envCython', 'np_version')
 
 # Qt build environment
 qt_env = env.Clone()
-qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "DBus", "Xml"]
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
 
 qt_libs = []
 if arch == "Darwin":
@@ -319,9 +312,6 @@ qt_flags = [
   "-DQT_NO_DEBUG",
   "-DQT_WIDGETS_LIB",
   "-DQT_GUI_LIB",
-  "-DQT_QUICK_LIB",
-  "-DQT_QUICKWIDGETS_LIB",
-  "-DQT_QML_LIB",
   "-DQT_CORE_LIB",
   "-DQT_MESSAGELOGCONTEXT",
 ]
@@ -346,24 +336,27 @@ Export('env', 'qt_env', 'arch', 'real_arch')
 SConscript(['common/SConscript'])
 Import('_common', '_gpucommon')
 
-common = [_common, 'json11']
+common = [_common, 'json11', 'zmq']
 gpucommon = [_gpucommon]
 
 Export('common', 'gpucommon')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
-SConscript(['msgq_repo/SConscript'])
+# Enable swaglog include in submodules
+env_swaglog = env.Clone()
+env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
+SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc/can/SConscript'], exports={'env': env_swaglog})
+
 SConscript(['cereal/SConscript'])
+
 Import('socketmaster', 'msgq')
 messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
 Export('messaging')
 
 
 # Build other submodules
-SConscript([
-  'opendbc/can/SConscript',
-  'panda/SConscript',
-])
+SConscript(['panda/SConscript'])
 
 # Build rednose library
 SConscript(['rednose/SConscript'])

cereal/SConscript

@@ -1,31 +1,20 @@
-Import('env', 'envCython', 'arch', 'common', 'msgq')
-
-import shutil
+Import('env', 'common', 'msgq')
 
 cereal_dir = Dir('.')
 gen_dir = Dir('gen')
-other_dir = Dir('#msgq')
 
 # Build cereal
 schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
-env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
 env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
             schema_files,
             f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
 
-# TODO: remove non shared cereal and messaging
-cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files])
-
-cereal = env.Library('cereal', cereal_objects)
-env.SharedLibrary('cereal_shared', cereal_objects)
+cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
-
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[msgq, 'zmq', common])
-
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[msgq, common])
 
-socketmaster = env.SharedObject(['messaging/socketmaster.cc'])
-socketmaster = env.Library('socketmaster', socketmaster)
+socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 
 Export('cereal', 'socketmaster')

cereal/messaging/__init__.py

@@ -92,26 +92,63 @@ def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader:
       return log_from_bytes(dat)
 
 
+class FrequencyTracker:
+  def __init__(self, service_freq: float, update_freq: float, is_poll: bool):
+    freq = max(min(service_freq, update_freq), 1.)
+    if is_poll:
+      min_freq = max_freq = freq
+    else:
+      max_freq = min(freq, update_freq)
+      if service_freq >= 2 * update_freq:
+        min_freq = update_freq
+      elif update_freq >= 2* service_freq:
+        min_freq = freq
+      else:
+        min_freq = min(freq, freq / 2.)
+
+    self.min_freq = min_freq * 0.8
+    self.max_freq = max_freq * 1.2
+    self.recv_dts: Deque[float] = deque(maxlen=int(10 * freq))
+    self.prev_time = 0.0
+
+  def record_recv_time(self, cur_time: float) -> None:
+    # TODO: Handle case where cur_time is less than prev_time
+    if self.prev_time > 1e-5:
+      self.recv_dts.append(cur_time - self.prev_time)
+    self.prev_time = cur_time
+
+  @property
+  def valid(self) -> bool:
+    if not self.recv_dts:
+      return False
+
+    avg_freq = len(self.recv_dts) / sum(self.recv_dts)
+    if self.min_freq <= avg_freq <= self.max_freq:
+      return True
+
+    recent_dts = list(self.recv_dts)[-int(self.recv_dts.maxlen / 10):]
+    avg_freq_recent = len(recent_dts) / sum(recent_dts)
+    return self.min_freq <= avg_freq_recent <= self.max_freq
+
+
 class SubMaster:
   def __init__(self, services: List[str], poll: Optional[str] = None,
                ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
                ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
     self.frame = -1
+    self.services = services
     self.seen = {s: False for s in services}
     self.updated = {s: False for s in services}
     self.recv_time = {s: 0. for s in services}
     self.recv_frame = {s: 0 for s in services}
     self.alive = {s: False for s in services}
     self.freq_ok = {s: False for s in services}
-    self.recv_dts: Dict[str, Deque[float]] = {}
     self.sock = {}
     self.data = {}
     self.valid = {}
     self.logMonoTime = {}
 
-    self.max_freq = {}
-    self.min_freq = {}
-
+    self.freq_tracker: Dict[str, FrequencyTracker] = {}
     self.poller = Poller()
     polled_services = set([poll, ] if poll is not None else services)
     self.non_polled_services = set(services) - polled_services
@@ -138,22 +175,7 @@ def __init__(self, services: List[str], poll: Optional[str] = None,
       self.data[s] = getattr(data.as_reader(), s)
       self.logMonoTime[s] = 0
       self.valid[s] = True  # FIXME: this should default to False
-
-      freq = max(min([SERVICE_LIST[s].frequency, self.update_freq]), 1.)
-      if s == poll:
-        max_freq = freq
-        min_freq = freq
-      else:
-        max_freq = min(freq, self.update_freq)
-        if SERVICE_LIST[s].frequency >= 2*self.update_freq:
-          min_freq = self.update_freq
-        elif self.update_freq >= 2*SERVICE_LIST[s].frequency:
-          min_freq = freq
-        else:
-          min_freq = min(freq, freq / 2.)
-      self.max_freq[s] = max_freq*1.2
-      self.min_freq[s] = min_freq*0.8
-      self.recv_dts[s] = deque(maxlen=int(10*freq))
+      self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
 
   def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
     return self.data[s]
@@ -173,7 +195,7 @@ def update(self, timeout: int = 100) -> None:
 
   def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
     self.frame += 1
-    self.updated = dict.fromkeys(self.updated, False)
+    self.updated = dict.fromkeys(self.services, False)
     for msg in msgs:
       if msg is None:
         continue
@@ -182,54 +204,30 @@ def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStruct
       self.seen[s] = True
       self.updated[s] = True
 
-      if self.recv_time[s] > 1e-5:
-        self.recv_dts[s].append(cur_time - self.recv_time[s])
+      self.freq_tracker[s].record_recv_time(cur_time)
       self.recv_time[s] = cur_time
       self.recv_frame[s] = self.frame
       self.data[s] = getattr(msg, s)
       self.logMonoTime[s] = msg.logMonoTime
       self.valid[s] = msg.valid
 
-    for s in self.data:
+    for s in self.services:
       if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation:
         # alive if delay is within 10x the expected frequency
         self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency)
-
-        # check average frequency; slow to fall, quick to recover
-        dts = self.recv_dts[s]
-        assert dts.maxlen is not None
-        recent_dts = list(dts)[-int(dts.maxlen / 10):]
-        try:
-          avg_freq = 1 / (sum(dts) / len(dts))
-          avg_freq_recent = 1 / (sum(recent_dts) / len(recent_dts))
-        except ZeroDivisionError:
-          avg_freq = 0
-          avg_freq_recent = 0
-
-        avg_freq_ok = self.min_freq[s] <= avg_freq <= self.max_freq[s]
-        recent_freq_ok = self.min_freq[s] <= avg_freq_recent <= self.max_freq[s]
-        self.freq_ok[s] = avg_freq_ok or recent_freq_ok
+        self.freq_ok[s] = self.freq_tracker[s].valid
       else:
         self.freq_ok[s] = True
-        if self.simulation:
-          self.alive[s] = self.seen[s] # alive is defined as seen when simulation flag set
-        else:
-          self.alive[s] = True
+        self.alive[s] = self.seen[s] if self.simulation else True
 
   def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
-    if service_list is None:
-      service_list = list(self.sock.keys())
-    return all(self.alive[s] for s in service_list if s not in self.ignore_alive)
+    return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
 
   def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
-    if service_list is None:
-      service_list = list(self.sock.keys())
-    return all(self.freq_ok[s] for s in service_list if self._check_avg_freq(s))
+    return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
 
   def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
-    if service_list is None:
-      service_list = list(self.sock.keys())
-    return all(self.valid[s] for s in service_list if s not in self.ignore_valid)
+    return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
 
   def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
     return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)

cereal/messaging/tests/test_pub_sub_master.py

@@ -89,8 +89,8 @@ def test_avg_frequency_checks(self):
       for service, (max_freq, min_freq) in checks.items():
         if max_freq is not None:
           assert sm._check_avg_freq(service)
-          assert sm.max_freq[service] == max_freq*1.2
-          assert sm.min_freq[service] == min_freq*0.8
+          assert sm.freq_tracker[service].max_freq == max_freq*1.2
+          assert sm.freq_tracker[service].min_freq == min_freq*0.8
         else:
           assert not sm._check_avg_freq(service)
 

common/gps.py

@@ -0,0 +1,8 @@
+from openpilot.common.params import Params
+
+
+def get_gps_location_service(params: Params) -> str:
+  if params.get_bool("UbloxAvailable"):
+    return "gpsLocationExternal"
+  else:
+    return "gpsLocation"

docs/car-porting/what-is-a-car-port.md

@@ -1,6 +1,6 @@
 # What is a car port?
 
-A car port enables openpilot support on a particular car. Each car model openpilot supports needs to be individually ported. All car ports live in `openpilot/selfdrive/car/`.
+A car port enables openpilot support on a particular car. Each car model openpilot supports needs to be individually ported. All car ports live in `openpilot/selfdrive/car/car_specific.py` and `opendbc_repo/opendbc/car`.
 
 The complexity of a car port varies depending on many factors including:
 * existing openpilot support for similar cars