Toyota: parse FW versions for more reliable fuzzy fingerprinting (com…

…maai#28641)

* add test from hyundai

* found tss2

* abs and engine are very rarely shared (avalonh tss2 and camryh tss2 is one example)

* some bad regex

* some pattern work

* some work

* .

* some conceptual clean up

* fix short fw pattern

* hyundai test

* clean up values.py a bit

* print platform codes

* hyundai fuzzy

* pass test

* move around constants

* clean up

* clean up hyundai

* fix print

* more clean up, fix med pattern

* add documentation

* use major version

* some clean up from merge

* some clean up from merge 2

* spot check

* clean up imports

* missing FW_QUERY_CONFIG

* short version: always prefixed with 3, get real platform code

* limit to max chunks seen (3)

* rm engine

* fix that

* get_platform_codes returns dict

* tests

* comments

* fix test

* enable a test

* fix script

* print ecu parts

* enable old test

* clean up some tests

* clean up

* more clean up

* static

* this is all it took?

* add note

* ...

* use less ECUs

* bump

* todo

* clean up fuzzy fp function

* make deterministic in a feat of engineering

* add temp exclude_fw argument for testing

* fix logic

* add blacklist

* add platform and its matches

* fix for nb

* remove fw exclusion

* Revert "remove fw exclusion"

This reverts commit 0e3b47c.

* clean up

* Revert "Revert "remove fw exclusion""

This reverts commit 42c55f0.

* these two have similar chassis according to wikipedia (but mass is relatively different)

* Revert "Revert "Revert "remove fw exclusion"""

This reverts commit 0f87423.

* Revert "Revert "Revert "Revert "remove fw exclusion""""

This reverts commit 2411967.

* oof

* shadows global variable

* rm comment

selfdrive/car/hyundai/values.py

@@ -400,7 +400,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]:
   # to distinguish between hybrid and ICE. All EVs so far are either exclusively
   # electric or specify electric in the platform code.
   # TODO: whitelist platforms that we've seen hybrid and ICE versions of that have these specifiers
-  fuzzy_platform_blacklist = {str(car) for car in set(CANFD_CAR - EV_CAR)}
+  fuzzy_platform_blacklist = {str(c) for c in set(CANFD_CAR - EV_CAR)}
   candidates: Set[str] = set()
 
   for candidate, fws in FW_VERSIONS.items():

selfdrive/car/toyota/tests/test_toyota.py

@@ -3,8 +3,10 @@
 import unittest
 
 from cereal import car
+from openpilot.selfdrive.car.fw_versions import build_fw_dict
 from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \
-                                                  PLATFORM_CODE_ECUS, get_platform_codes
+                                                  FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, FUZZY_EXCLUDED_PLATFORMS, \
+                                                  get_platform_codes
 
 Ecu = car.CarParams.Ecu
 ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()}
@@ -120,6 +122,27 @@ def test_platform_codes_spot_check(self):
     ])
     self.assertEqual(results, {b"F1526-07-1": {b"10", b"40"}, b"8646F-41-04": {b"100"}, b"58-79": {b"000"}})
 
+  def test_fuzzy_excluded_platforms(self):
+    # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared.
+    platforms_with_shared_codes = set()
+    for platform, fw_by_addr in FW_VERSIONS.items():
+      car_fw = []
+      for ecu, fw_versions in fw_by_addr.items():
+        ecu_name, addr, sub_addr = ecu
+        for fw in fw_versions:
+          car_fw.append({"ecu": ecu_name, "fwVersion": fw, "address": addr,
+                         "subAddress": 0 if sub_addr is None else sub_addr})
+
+      CP = car.CarParams.new_message(carFw=car_fw)
+      matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw))
+      if len(matches) == 1:
+        self.assertEqual(list(matches)[0], platform)
+      else:
+        # If a platform has multiple matches, add it and its matches
+        platforms_with_shared_codes |= {platform, *matches}
+
+    self.assertEqual(platforms_with_shared_codes, FUZZY_EXCLUDED_PLATFORMS, (len(platforms_with_shared_codes), len(FW_VERSIONS)))
+
 
 if __name__ == "__main__":
   unittest.main()

selfdrive/car/toyota/values.py

@@ -284,6 +284,40 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]:
   return dict(codes)
 
 
+def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]:
+  candidates = set()
+
+  for candidate, fws in FW_VERSIONS.items():
+    # Keep track of ECUs which pass all checks (platform codes, within sub-version range)
+    valid_found_ecus = set()
+    valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS}
+    for ecu, expected_versions in fws.items():
+      addr = ecu[1:]
+      # Only check ECUs expected to have platform codes
+      if ecu[0] not in PLATFORM_CODE_ECUS:
+        continue
+
+      # Expected platform codes & versions
+      expected_platform_codes = get_platform_codes(expected_versions)
+
+      # Found platform codes & versions
+      found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set()))
+
+      # Check part number + platform code + major version matches for any found versions
+      # Platform codes and major versions change for different physical parts, generation, API, etc.
+      # Sub-versions are incremented for minor recalls, do not need to be checked.
+      if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes):
+        break
+
+      valid_found_ecus.add(addr)
+
+    # If all live ECUs pass all checks for candidate, add it as a match
+    if valid_expected_ecus.issubset(valid_found_ecus):
+      candidates.add(candidate)
+
+  return {str(c) for c in (candidates - FUZZY_EXCLUDED_PLATFORMS)}
+
+
 # Regex patterns for parsing more general platform-specific identifiers from FW versions.
 # - Part number: Toyota part number (usually last character needs to be ignored to find a match).
 #    Each ECU address has just one part number.
@@ -311,6 +345,8 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]:
 # - eps: describes lateral API changes for the EPS, such as using LTA for lane keeping and rejecting LKA messages
 PLATFORM_CODE_ECUS = [Ecu.fwdCamera, Ecu.abs, Ecu.eps]
 
+# These platforms have at least one platform code for all ECUs shared with another platform.
+FUZZY_EXCLUDED_PLATFORMS = {CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2}
 
 # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers.
 # Toyota diagnostic software first gets the supported data ids, then queries them one by one.
@@ -382,6 +418,7 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]:
     (Ecu.combinationMeter, 0x7c0, None),
     (Ecu.hvac, 0x7c4, None),
   ],
+  match_fw_to_car_fuzzy=match_fw_to_car_fuzzy,
 )
 
 FW_VERSIONS = {