Merge commit '3ef35ed2298a3a9d199f9145409547710065884c' into subaru-c…

…ommunity

.github/workflows/tests.yml

@@ -3,7 +3,7 @@ name: tests
 on: [push, pull_request]
 
 env:
-  RUN: docker run -v $GITHUB_WORKSPACE:/project/opendbc -w /project/opendbc -e PYTHONWARNINGS=error --shm-size 1G --rm opendbc /bin/bash -c
+  RUN: docker run -v $GITHUB_WORKSPACE:/project/opendbc -w /project/opendbc -e PYTHONWARNINGS="error,default::DeprecationWarning" --shm-size 1G --rm opendbc /bin/bash -c
   BUILD: |
       docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile) || true
       docker pull ghcr.io/commaai/opendbc:latest || true

.pre-commit-config.yaml

@@ -7,15 +7,15 @@ repos:
     -   id: check-merge-conflict
     -   id: check-symlinks
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.4
+    rev: v2.2.5
     hooks:
     -   id: codespell
         exclude: '\.dbc$'
         args:
         - --check-hidden
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.4.1
     hooks:
     -   id: mypy
 -   repo: https://github.com/PyCQA/flake8

Dockerfile

@@ -34,13 +34,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-RUN pyenv install 3.8.10
-RUN pyenv global 3.8.10
+RUN pyenv install 3.11.4
+RUN pyenv global 3.11.4
 RUN pyenv rehash
 
 COPY requirements.txt /tmp/
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
-RUN pip install --no-cache-dir pre-commit==2.15.0 pylint==2.5.2
+RUN pip install --no-cache-dir pre-commit==2.15.0 pylint==2.17.4
 
 ENV PYTHONPATH=/project
 

can/packer.cc

@@ -1,8 +1,9 @@
-#include <cassert>
-#include <utility>
 #include <algorithm>
-#include <map>
+#include <cassert>
 #include <cmath>
+#include <map>
+#include <stdexcept>
+#include <utility>
 
 #include "opendbc/can/common.h"
 

can/packer_pyx.pyx

@@ -14,8 +14,7 @@ cdef class CANPacker:
   cdef:
     cpp_CANPacker *packer
     const DBC *dbc
-    map[string, (int, int)] name_to_address_and_size
-    map[int, int] address_to_size
+    map[string, int] name_to_address
 
   def __init__(self, dbc_name):
     self.dbc = dbc_lookup(dbc_name)
@@ -25,8 +24,7 @@ cdef class CANPacker:
     self.packer = new cpp_CANPacker(dbc_name)
     for i in range(self.dbc[0].msgs.size()):
       msg = self.dbc[0].msgs[i]
-      self.name_to_address_and_size[string(msg.name)] = (msg.address, msg.size)
-      self.address_to_size[msg.address] = msg.size
+      self.name_to_address[string(msg.name)] = msg.address
 
   cdef vector[uint8_t] pack(self, addr, values):
     cdef vector[SignalPackValue] values_thing
@@ -41,12 +39,11 @@ cdef class CANPacker:
     return self.packer.pack(addr, values_thing)
 
   cpdef make_can_msg(self, name_or_addr, bus, values):
-    cdef int addr, size
+    cdef int addr
     if type(name_or_addr) == int:
       addr = name_or_addr
-      size = self.address_to_size[name_or_addr]
     else:
-      addr, size = self.name_to_address_and_size[name_or_addr.encode("utf8")]
+      addr = self.name_to_address[name_or_addr.encode("utf8")]
 
     cdef vector[uint8_t] val = self.pack(addr, values)
-    return [addr, 0, (<char *>&val[0])[:size], bus]
+    return [addr, 0, (<char *>&val[0])[:val.size()], bus]

can/parser_pyx.pyx

@@ -41,12 +41,17 @@ cdef class CANParser:
     self.vl_all = {}
     self.ts_nanos = {}
     msg_name_to_address = {}
+    msg_address_to_signals = {}
 
     for i in range(self.dbc[0].msgs.size()):
       msg = self.dbc[0].msgs[i]
       name = msg.name.decode("utf8")
 
       msg_name_to_address[name] = msg.address
+      msg_address_to_signals[msg.address] = set()
+      for sig in msg.sigs:
+        msg_address_to_signals[msg.address].add(sig.name.decode("utf8"))
+
       self.address_to_msg_name[msg.address] = name
       self.vl[msg.address] = {}
       self.vl[name] = self.vl[msg.address]
@@ -58,12 +63,13 @@ cdef class CANParser:
     # Convert message names into addresses
     for i in range(len(signals)):
       s = signals[i]
-      if not isinstance(s[1], numbers.Number):
-        if s[1] not in msg_name_to_address:
-          print(msg_name_to_address)
-          raise RuntimeError(f"could not find message {repr(s[1])} in DBC {self.dbc_name}")
-        s = (s[0], msg_name_to_address[s[1]])
-        signals[i] = s
+      address = s[1] if isinstance(s[1], numbers.Number) else msg_name_to_address.get(s[1])
+      if address not in msg_address_to_signals:
+        raise RuntimeError(f"could not find message {repr(s[1])} in DBC {self.dbc_name}")
+      if s[0] not in msg_address_to_signals[address]:
+        raise RuntimeError(f"could not find signal {repr(s[0])} in {repr(s[1])}, DBC {self.dbc_name}")
+
+      signals[i] = (s[0], address)
 
     for i in range(len(checks)):
       c = checks[i]

can/tests/__init__.py

@@ -0,0 +1,8 @@
+import glob
+import os
+
+from opendbc import DBC_PATH
+
+ALL_DBCS = [os.path.basename(dbc).split('.')[0] for dbc in
+            glob.glob(f"{DBC_PATH}/*.dbc")]
+TEST_DBC = os.path.abspath(os.path.join(os.path.dirname(__file__), "test.dbc"))

can/tests/test.dbc

@@ -23,3 +23,5 @@ BO_ 245 CAN_FD_MESSAGE: 32 XXX
  SG_ SIGNED : 22|16@0- (1,0) [0|1] "" XXX
  SG_ 64_BIT_LE : 159|64@1+ (1,0) [0|1] "" XXX
  SG_ 64_BIT_BE : 80|64@0+ (1,0) [0|1] "" XXX
+
+VAL_ 80 NON_EXISTENT_ADDR 0 "test";

can/tests/test_dbc_exceptions.py

@@ -4,6 +4,8 @@
 
 from opendbc.can.parser import CANParser, CANDefine
 from opendbc.can.packer import CANPacker
+from opendbc.can.tests import TEST_DBC
+
 
 class TestCanParserPackerExceptions(unittest.TestCase):
   def test_civic_exceptions(self):
@@ -20,6 +22,8 @@ def test_civic_exceptions(self):
       CANPacker(dbc_invalid)
     with self.assertRaises(RuntimeError):
       CANDefine(dbc_invalid)
+    with self.assertRaises(KeyError):
+      CANDefine(TEST_DBC)
 
     with self.assertRaises(RuntimeError):
       CANParser(dbc_file, signals, [], 0)

can/tests/test_dbc_parser.py

@@ -1,19 +1,11 @@
 #!/usr/bin/env python3
-import glob
-import os
 import unittest
 
-from opendbc import DBC_PATH
 from opendbc.can.parser import CANParser
+from opendbc.can.tests import ALL_DBCS
 
 
 class TestDBCParser(unittest.TestCase):
-  @classmethod
-  def setUpClass(cls):
-    cls.dbcs = []
-    for dbc in glob.glob(f"{DBC_PATH}/*.dbc"):
-      cls.dbcs.append(os.path.basename(dbc).split('.')[0])
-
   def test_parse_all_dbcs(self):
     """
       Dynamic DBC parser checks:
@@ -23,7 +15,7 @@ def test_parse_all_dbcs(self):
         - All BO_, SG_, VAL_ lines for syntax errors
     """
 
-    for dbc in self.dbcs:
+    for dbc in ALL_DBCS:
       with self.subTest(dbc=dbc):
         CANParser(dbc, [], [], 0)
 

can/tests/test_define.py

@@ -2,6 +2,7 @@
 import unittest
 
 from opendbc.can.can_define import CANDefine
+from opendbc.can.tests import ALL_DBCS
 
 
 class TestCADNDefine(unittest.TestCase):
@@ -22,6 +23,12 @@ def test_civic(self):
                           }
                          )
 
+  def test_all_dbcs(self):
+    # Asserts no exceptions on all DBCs
+    for dbc in ALL_DBCS:
+      with self.subTest(dbc=dbc):
+        CANDefine(dbc)
+
 
 if __name__ == "__main__":
   unittest.main()

can/tests/test_packer_parser.py

@@ -1,14 +1,11 @@
 #!/usr/bin/env python3
-import os
 import unittest
 import random
 
 import cereal.messaging as messaging
 from opendbc.can.parser import CANParser
 from opendbc.can.packer import CANPacker
-
-
-TEST_DBC = os.path.abspath(os.path.join(os.path.dirname(__file__), "test.dbc"))
+from opendbc.can.tests import TEST_DBC
 
 
 # Python implementation so we don't have to depend on boardd
@@ -317,6 +314,24 @@ def test_timestamp_nanos(self):
       ts_nanos = parser.ts_nanos["POWERTRAIN_DATA"].values()
       self.assertEqual(set(ts_nanos), {0})
 
+  def test_undefined_signals(self):
+    # Ensure we don't allow messages or signals not in the DBC
+    existing_signals = {
+      "STEERING_CONTROL": ["STEER_TORQUE_REQUEST", "SET_ME_X00_2", "COUNTER"],
+      228: ["STEER_TORQUE_REQUEST", "SET_ME_X00_2", "COUNTER"],
+      "CAN_FD_MESSAGE": ["SIGNED", "64_BIT_LE", "64_BIT_BE", "COUNTER"],
+      245: ["SIGNED", "64_BIT_LE", "64_BIT_BE", "COUNTER"],
+    }
+
+    for msg, sigs in existing_signals.items():
+      for sig in sigs:
+        CANParser(TEST_DBC, [(sig, msg)], [(msg, 0)])
+        new_msg = msg + "1" if isinstance(msg, str) else msg + 1
+        self.assertRaises(RuntimeError, CANParser, TEST_DBC, [(sig + "1", msg)], [(msg, 0)])
+        self.assertRaises(RuntimeError, CANParser, TEST_DBC, [(sig, new_msg)], [(msg, 0)])
+        self.assertRaises(RuntimeError, CANParser, TEST_DBC, [(sig, msg)], [(new_msg, 0)])
+        self.assertRaises(RuntimeError, CANParser, TEST_DBC, [(sig, new_msg)], [(new_msg, 0)])
+
 
 if __name__ == "__main__":
   unittest.main()

generator/chrysler/_stellantis_common_ram.py

@@ -32,7 +32,7 @@
   chrysler_path = os.path.dirname(os.path.realpath(__file__))
 
   for out, addr_lookup in chrysler_to_ram.items():
-    with open(os.path.join(chrysler_path, src)) as in_f, open(os.path.join(chrysler_path, out), 'w') as out_f:
+    with open(os.path.join(chrysler_path, src), encoding='utf-8') as in_f, open(os.path.join(chrysler_path, out), 'w', encoding='utf-8') as out_f:
       out_f.write(f'CM_ "Generated from {src}"\n\n')
 
       wrote_addrs = set()

generator/generator.py

@@ -11,7 +11,7 @@
 
 
 def read_dbc(src_dir: str, filename: str) -> str:
-  with open(os.path.join(src_dir, filename)) as file_in:
+  with open(os.path.join(src_dir, filename), encoding='utf-8') as file_in:
     return file_in.read()
 
 
@@ -23,7 +23,7 @@ def create_dbc(src_dir: str, filename: str, output_path: str):
   output_filename = filename.replace('.dbc', generated_suffix)
   output_file_location = os.path.join(output_path, output_filename)
 
-  with open(output_file_location, 'w') as dbc_file_out:
+  with open(output_file_location, 'w', encoding='utf-8') as dbc_file_out:
     dbc_file_out.write('CM_ "AUTOGENERATED FILE, DO NOT EDIT";\n')
 
     for include_filename in includes:

generator/hyundai/hyundai_kia_mando_corner_radar.py

@@ -5,7 +5,7 @@
 if __name__ == "__main__":
   dbc_name = os.path.basename(__file__).replace(".py", ".dbc")
   hyundai_path = os.path.dirname(os.path.realpath(__file__))
-  with open(os.path.join(hyundai_path, dbc_name), "w") as f:
+  with open(os.path.join(hyundai_path, dbc_name), "w", encoding='utf-8') as f:
     f.write("""
 VERSION ""
 

generator/hyundai/hyundai_kia_mando_front_radar.py

@@ -4,7 +4,7 @@
 if __name__ == "__main__":
   dbc_name = os.path.basename(__file__).replace(".py", ".dbc")
   hyundai_path = os.path.dirname(os.path.realpath(__file__))
-  with open(os.path.join(hyundai_path, dbc_name), "w") as f:
+  with open(os.path.join(hyundai_path, dbc_name), "w", encoding='utf-8') as f:
     f.write("""
 VERSION ""
 

generator/subaru/_subaru_global.dbc

@@ -83,10 +83,14 @@ BO_ 314 Wheel_Speeds: 8 XXX
  SG_ FL : 51|13@1+ (0.057,0) [0|255] "kph" XXX
  SG_ RL : 38|13@1+ (0.057,0) [0|255] "kph" XXX
 
-BO_ 280 STOP_START: 8 XXX
+BO_ 280 Steering_Torque_2: 8 XXX
  SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX
  SG_ COUNTER : 8|4@1+ (1,0) [0|15] "" XXX
- SG_ State : 63|1@1+ (1,0) [0|1] "" XXX
+ SG_ Steer_Torque_Output : 13|11@1- (-10,0) [0|255] "" XXX
+ SG_ Signal1 : 24|8@1+ (1,0) [0|511] "" XXX
+ SG_ Steer_Torque_Sensor : 45|11@1- (-1,0) [0|255] "" XXX
+ SG_ Steering_Active : 61|1@0+ (1,0) [0|1] "" XXX
+ SG_ Steering_Disabled : 63|1@1+ (1,0) [0|1] "" XXX
 
 BO_ 281 Steering_Torque: 8 XXX
  SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX
@@ -98,6 +102,11 @@ BO_ 281 Steering_Torque: 8 XXX
  SG_ Steering_Angle : 32|16@1- (-0.0217,0) [-600|600] "" X
  SG_ Steer_Torque_Output : 48|11@1- (-10,0) [-1000|1000] "" XXX
 
+BO_ 282 Steering_2: 8 XXX
+ SG_ CHECKSUM : 0|8@1+ (1,0) [0|1] "" XXX
+ SG_ COUNTER : 8|4@1+ (1,0) [0|1] "" XXX
+ SG_ Steering_Angle : 24|17@1- (-0.01,0) [0|1] "" XXX
+
 BO_ 312 Brake_Pressure_L_R: 8 XXX
  SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX
  SG_ COUNTER : 8|4@1+ (1,0) [0|15] "" XXX
@@ -127,12 +136,19 @@ BO_ 290 ES_LKAS: 8 XXX
  SG_ LKAS_Output : 16|13@1- (-1,0) [-8191|8191] "" XXX
  SG_ LKAS_Request : 29|1@0+ (1,0) [0|1] "" XXX
 
+BO_ 292 ES_LKAS_ALT: 8 XXX
+ SG_ CHECKSUM : 0|8@1+ (1,0) [0|1] "" XXX
+ SG_ COUNTER : 8|4@1+ (1,0) [0|1] "" XXX
+ SG_ LKAS_Request : 12|1@1+ (1,0) [0|1] "" XXX
+ SG_ LKAS_Output : 40|17@1- (-1,0) [0|1] "" XXX
+ SG_ SET_3 : 60|2@1+ (1,0) [0|1] "" XXX
+
 BO_ 544 ES_Brake: 8 XXX
  SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX
  SG_ COUNTER : 8|4@1+ (1,0) [0|15] "" XXX
  SG_ Signal1 : 12|4@1+ (1,0) [0|15] "" XXX
  SG_ Brake_Pressure : 16|16@1+ (1,0) [0|65535] "" XXX
- SG_ Signal2 : 32|4@1+ (1,0) [0|15] "" XXX
+ SG_ AEB_Status : 32|4@1+ (1,0) [0|15] "" XXX
  SG_ Cruise_Brake_Lights : 36|1@1+ (1,0) [0|1] "" XXX
  SG_ Cruise_Brake_Fault : 37|1@1+ (1,0) [0|1] "" XXX
  SG_ Cruise_Brake_Active : 38|1@1+ (1,0) [0|1] "" XXX
@@ -240,6 +256,9 @@ BO_ 1677 Dash_State: 8 XXX
 CM_ SG_ 64 Throttle_Combo "Throttle Cruise + Pedal";
 CM_ SG_ 313 Brake_Lights "Driver or Cruise Brake on";
 CM_ SG_ 544 Cruise_Brake_Lights "1 = switch on brake lights";
+CM_ SG_ 544 Brake_Pressure "Winds down after cruise disabled. Also can be non-zero when likely preparing for AEB";
+CM_ SG_ 544 Signal3 "Usually goes to 2 if AEB_Status is 4";
+CM_ SG_ 544 AEB_Status "Occasionally is 4 instead of 8 while Brake_Pressure is non-zero, unsure why";
 CM_ SG_ 801 PCB_Off "Pre-Collision Braking off";
 CM_ SG_ 801 Brake_Lights "Driver or Cruise brake on";
 CM_ SG_ 801 Cruise_State "0 = Normal, 1 = Hold+User Brake, 2 = Ready, 3 = Hold";
@@ -256,3 +275,4 @@ CM_ SG_ 802 LKAS_Dash_State "0 = Off, 1 = Ready, 2 = Active";
 CM_ SG_ 802 LKAS_Right_Line_Visible "0 = Off, 1 = White, 2 = Green, 3 = Orange";
 CM_ SG_ 912 UNITS "0 = Metric, 1 = Imperial";
 CM_ SG_ 912 ICY_ROAD "1 = DASHLIGHT ON, 2 = WARNING, 3 = OFF";
+VAL_ 544 AEB_Status 12 "AEB related" 8 "AEB actuation" 4 "AEB related" 0 "No AEB actuation";

generator/subaru/subaru_global_2017.dbc

@@ -53,7 +53,7 @@ BO_ 576 CruiseControl: 8 XXX
  SG_ Cruise_Activated : 41|1@1+ (1,0) [0|1] "" XXX
  SG_ Signal2 : 42|22@1+ (1,0) [0|4194303] "" XXX
 
-BO_ 803 INFOTAINMENT_STATUS: 8 XXX
+BO_ 803 ES_Infotainment: 8 XXX
  SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX
  SG_ COUNTER : 8|4@1+ (1,0) [0|15] "" XXX
  SG_ LKAS_Blue_Lines : 15|4@0+ (1,0) [0|15] "" XXX