Merge commit 'ed8ff7e4' into subaru-community

Jenkinsfile

@@ -82,7 +82,7 @@ pipeline {
           steps {
             phone_steps("panda-dos", [
               ["build", "scons -j4"],
-              ["flash", "cd tests/ && ./ci_reset_internal_hw.py"],
+              ["flash", "cd tests/ && ./reflash_internal_panda.py"],
               ["test", "cd tests/hitl && HW_TYPES=6 pytest --durations=0 [2-7]*.py -k 'not test_send_recv'"],
             ])
           }
@@ -93,7 +93,7 @@ pipeline {
           steps {
             phone_steps("panda-tres", [
               ["build", "scons -j4"],
-              ["flash", "cd tests/ && ./ci_reset_internal_hw.py"],
+              ["flash", "cd tests/ && ./reflash_internal_panda.py"],
               ["test", "cd tests/hitl && HW_TYPES=9 pytest --durations=0 2*.py [5-9]*.py"],
             ])
           }

__init__.py

@@ -1,6 +1,6 @@
-from .python.constants import McuType, BASEDIR, FW_PATH  # noqa: F401
+from .python.constants import McuType, BASEDIR, FW_PATH, USBPACKET_MAX_SIZE  # noqa: F401
 from .python.spi import PandaSpiException, PandaProtocolMismatch  # noqa: F401
 from .python.serial import PandaSerial  # noqa: F401
 from .python import (Panda, PandaDFU, # noqa: F401
                      pack_can_buffer, unpack_can_buffer, calculate_checksum, unpack_log,
-                     DLC_TO_LEN, LEN_TO_DLC, ALTERNATIVE_EXPERIENCE, USBPACKET_MAX_SIZE, CANPACKET_HEAD_SIZE)
+                     DLC_TO_LEN, LEN_TO_DLC, ALTERNATIVE_EXPERIENCE, CANPACKET_HEAD_SIZE)

board/README.md

@@ -4,16 +4,17 @@ Programming
 **Panda**
 
 ```
-./recover.py      # flash bootstub
 ./flash.py        # flash application
+./recover.py      # flash bootstub
 ```
 
 Troubleshooting
 ----
 
 If your panda will not flash and green LED is on, use `recover.py`.
 If panda is blinking fast with green LED, use `flash.py`.
+
 Otherwise if LED is off and panda can't be seen with `lsusb` command, use [panda paw](https://comma.ai/shop/products/panda-paw) to go into DFU mode.
 
 
-[dfu-util](http://github.com/dsigma/dfu-util.git) for flashing
+If your device has an internal panda and none of the above works, try running `../tests/reflash_internal_panda.py`.

board/drivers/gmlan_alt.h

@@ -272,6 +272,7 @@ bool bitbang_gmlan(CANPacket_t *to_bang) {
   gmlan_send_ok = true;
   gmlan_alt_mode = BITBANG;
 
+#ifndef STM32H7
   if (gmlan_sendmax == -1) {
     int len = get_bit_message(pkt_stuffed, to_bang);
     gmlan_fail_count = 0;
@@ -285,5 +286,8 @@ bool bitbang_gmlan(CANPacket_t *to_bang) {
     // 33kbps
     setup_timer();
   }
+#else
+  UNUSED(to_bang);
+#endif
   return gmlan_send_ok;
 }

board/drivers/spi.h

@@ -84,7 +84,7 @@ uint16_t spi_version_packet(uint8_t *out) {
   data_len += 1U;
 
   // SPI protocol version
-  out[data_pos + data_len] = 0x1;
+  out[data_pos + data_len] = 0x2;
   data_len += 1U;
 
   // data length

board/safety/safety_subaru.h

@@ -1,24 +1,18 @@
-const SteeringLimits SUBARU_STEERING_LIMITS = {
-  .max_steer = 2047,
-  .max_rt_delta = 940,
-  .max_rt_interval = 250000,
-  .max_rate_up = 50,
-  .max_rate_down = 70,
-  .driver_torque_factor = 50,
-  .driver_torque_allowance = 60,
-  .type = TorqueDriverLimited,
-};
+#define SUBARU_STEERING_LIMITS_GENERATOR(steer_max, rate_up, rate_down)          \
+  {                                                                              \
+    .max_steer = (steer_max),                                                    \
+    .max_rt_delta = 940,                                                         \
+    .max_rt_interval = 250000,                                                   \
+    .max_rate_up = (rate_up),                                                    \
+    .max_rate_down = (rate_down),                                                \
+    .driver_torque_factor = 50,                                                  \
+    .driver_torque_allowance = 60,                                               \
+    .type = TorqueDriverLimited,                                                 \
+  }                                                                              \
+
+const SteeringLimits SUBARU_STEERING_LIMITS = SUBARU_STEERING_LIMITS_GENERATOR(2047, 50, 70);
+const SteeringLimits SUBARU_GEN2_STEERING_LIMITS = SUBARU_STEERING_LIMITS_GENERATOR(1000, 40, 40);
 
-const SteeringLimits SUBARU_GEN2_STEERING_LIMITS = {
-  .max_steer = 1000,
-  .max_rt_delta = 940,
-  .max_rt_interval = 250000,
-  .max_rate_up = 40,
-  .max_rate_down = 40,
-  .driver_torque_factor = 50,
-  .driver_torque_allowance = 60,
-  .type = TorqueDriverLimited,
-};
 
 #define MSG_SUBARU_Brake_Status          0x13c
 #define MSG_SUBARU_CruiseControl         0x240

python/__init__.py

@@ -26,7 +26,6 @@
 LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
 logging.basicConfig(level=LOGLEVEL, format='%(message)s')
 
-USBPACKET_MAX_SIZE = 0x40
 CANPACKET_HEAD_SIZE = 0x6
 DLC_TO_LEN = [0, 1, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20, 24, 32, 48, 64]
 LEN_TO_DLC = {length: dlc for (dlc, length) in enumerate(DLC_TO_LEN)}

python/base.py

@@ -1,5 +1,4 @@
 from abc import ABC, abstractmethod
-from typing import List
 
 from .constants import McuType
 
@@ -24,7 +23,7 @@ def controlRead(self, request_type: int, request: int, value: int, index: int, l
     ...
 
   @abstractmethod
-  def bulkWrite(self, endpoint: int, data: List[int], timeout: int = TIMEOUT) -> int:
+  def bulkWrite(self, endpoint: int, data: bytes, timeout: int = TIMEOUT) -> int:
     ...
 
   @abstractmethod

python/constants.py

@@ -5,6 +5,8 @@
 BASEDIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")
 FW_PATH = os.path.join(BASEDIR, "board/obj/")
 
+USBPACKET_MAX_SIZE = 0x40
+
 class McuConfig(NamedTuple):
   mcu: str
   mcu_idcode: int

python/spi.py

@@ -9,10 +9,10 @@
 import threading
 from contextlib import contextmanager
 from functools import reduce
-from typing import List, Optional
+from typing import Callable, List, Optional
 
 from .base import BaseHandle, BaseSTBootloaderHandle, TIMEOUT
-from .constants import McuType, MCU_TYPE_BY_IDCODE
+from .constants import McuType, MCU_TYPE_BY_IDCODE, USBPACKET_MAX_SIZE
 from .utils import crc8_pedal
 
 try:
@@ -32,7 +32,7 @@
 MIN_ACK_TIMEOUT_MS = 100
 MAX_XFER_RETRY_COUNT = 5
 
-XFER_SIZE = 1000
+XFER_SIZE = 0x40*31
 
 DEV_PATH = "/dev/spidev0.0"
 
@@ -113,15 +113,15 @@ class PandaSpiHandle(BaseHandle):
   A class that mimics a libusb1 handle for panda SPI communications.
   """
 
-  PROTOCOL_VERSION = 1
+  PROTOCOL_VERSION = 2
 
-  def __init__(self):
+  def __init__(self) -> None:
     self.dev = SpiDevice()
 
-    self._transfer = self._transfer_spidev
+    self._transfer_raw: Callable[[SpiDevice, int, bytes, int, int, bool], bytes] = self._transfer_spidev
 
     if "KERN" in os.environ:
-      self._transfer = self._transfer_kernel_driver
+      self._transfer_raw = self._transfer_kernel_driver
 
       self.tx_buf = bytearray(1024)
       self.rx_buf = bytearray(1024)
@@ -134,25 +134,65 @@ def __init__(self):
       self.fileno = self.dev._spidev.fileno()
 
   # helpers
-  def _calc_checksum(self, data: List[int]) -> int:
+  def _calc_checksum(self, data: bytes) -> int:
     cksum = CHECKSUM_START
     for b in data:
       cksum ^= b
     return cksum
 
-  def _wait_for_ack(self, spi, ack_val: int, timeout: int, tx: int) -> None:
+  def _wait_for_ack(self, spi, ack_val: int, timeout: int, tx: int, length: int = 1) -> bytes:
     timeout_s = max(MIN_ACK_TIMEOUT_MS, timeout) * 1e-3
 
     start = time.monotonic()
     while (timeout == 0) or ((time.monotonic() - start) < timeout_s):
-      dat = spi.xfer2([tx, ])[0]
-      if dat == NACK:
+      dat = spi.xfer2([tx, ] * length)
+      if dat[0] == NACK:
         raise PandaSpiNackResponse
-      elif dat == ack_val:
-        return
+      elif dat[0] == ack_val:
+        return bytes(dat)
 
     raise PandaSpiMissingAck
 
+  def _transfer_spidev(self, spi, endpoint: int, data, timeout: int, max_rx_len: int = 1000, expect_disconnect: bool = False) -> bytes:
+    max_rx_len = max(USBPACKET_MAX_SIZE, max_rx_len)
+
+    logging.debug("- send header")
+    packet = struct.pack("<BBHH", SYNC, endpoint, len(data), max_rx_len)
+    packet += bytes([self._calc_checksum(packet), ])
+    spi.xfer2(packet)
+
+    logging.debug("- waiting for header ACK")
+    self._wait_for_ack(spi, HACK, MIN_ACK_TIMEOUT_MS, 0x11)
+
+    logging.debug("- sending data")
+    packet = bytes([*data, self._calc_checksum(data)])
+    spi.xfer2(packet)
+
+    if expect_disconnect:
+      logging.debug("- expecting disconnect, returning")
+      return b""
+    else:
+      logging.debug("- waiting for data ACK")
+      preread_len = USBPACKET_MAX_SIZE + 1  # read enough for a controlRead
+      dat = self._wait_for_ack(spi, DACK, timeout, 0x13, length=3 + preread_len)
+
+      # get response length, then response
+      response_len = struct.unpack("<H", dat[1:3])[0]
+      if response_len > max_rx_len:
+        raise PandaSpiException(f"response length greater than max ({max_rx_len} {response_len})")
+
+      # read rest
+      remaining = (response_len + 1) - preread_len
+      if remaining > 0:
+        dat += bytes(spi.readbytes(remaining))
+
+
+      dat = dat[:3 + response_len + 1]
+      if self._calc_checksum(dat) != 0:
+        raise PandaSpiBadChecksum
+
+      return dat[3:-1]
+
   def _transfer_kernel_driver(self, spi, endpoint: int, data, timeout: int, max_rx_len: int = 1000, expect_disconnect: bool = False) -> bytes:
     self.tx_buf[:len(data)] = data
     self.ioctl_data.endpoint = endpoint
@@ -169,54 +209,22 @@ def _transfer_kernel_driver(self, spi, endpoint: int, data, timeout: int, max_rx
       raise PandaSpiException(f"ioctl returned {ret}")
     return bytes(self.rx_buf[:ret])
 
-  def _transfer_spidev(self, spi, endpoint: int, data, timeout: int, max_rx_len: int = 1000, expect_disconnect: bool = False) -> bytes:
+  def _transfer(self, endpoint: int, data, timeout: int, max_rx_len: int = 1000, expect_disconnect: bool = False) -> bytes:
     logging.debug("starting transfer: endpoint=%d, max_rx_len=%d", endpoint, max_rx_len)
     logging.debug("==============================================")
 
     n = 0
     start_time = time.monotonic()
     exc = PandaSpiException()
-    while (time.monotonic() - start_time) < timeout*1e-3:
+    while (timeout == 0) or (time.monotonic() - start_time) < timeout*1e-3:
       n += 1
       logging.debug("\ntry #%d", n)
-      try:
-        logging.debug("- send header")
-        packet = struct.pack("<BBHH", SYNC, endpoint, len(data), max_rx_len)
-        packet += bytes([reduce(lambda x, y: x^y, packet) ^ CHECKSUM_START])
-        spi.xfer2(packet)
-
-        to = timeout - (time.monotonic() - start_time)*1e3
-        logging.debug("- waiting for header ACK")
-        self._wait_for_ack(spi, HACK, int(to), 0x11)
-
-        # send data
-        logging.debug("- sending data")
-        packet = bytes([*data, self._calc_checksum(data)])
-        spi.xfer2(packet)
-
-        if expect_disconnect:
-          logging.debug("- expecting disconnect, returning")
-          return b""
-        else:
-          to = timeout - (time.monotonic() - start_time)*1e3
-          logging.debug("- waiting for data ACK")
-          self._wait_for_ack(spi, DACK, int(to), 0x13)
-
-          # get response length, then response
-          response_len_bytes = bytes(spi.xfer2(b"\x00" * 2))
-          response_len = struct.unpack("<H", response_len_bytes)[0]
-          if response_len > max_rx_len:
-            raise PandaSpiException("response length greater than max")
-
-          logging.debug("- receiving response")
-          dat = bytes(spi.xfer2(b"\x00" * (response_len + 1)))
-          if self._calc_checksum([DACK, *response_len_bytes, *dat]) != 0:
-            raise PandaSpiBadChecksum
-
-          return dat[:-1]
-      except PandaSpiException as e:
-        exc = e
-        logging.debug("SPI transfer failed, retrying", exc_info=True)
+      with self.dev.acquire() as spi:
+        try:
+          return self._transfer_raw(spi, endpoint, data, timeout, max_rx_len, expect_disconnect)
+        except PandaSpiException as e:
+          exc = e
+          logging.debug("SPI transfer failed, retrying", exc_info=True)
 
     raise exc
 
@@ -261,29 +269,24 @@ def close(self):
     self.dev.close()
 
   def controlWrite(self, request_type: int, request: int, value: int, index: int, data, timeout: int = TIMEOUT, expect_disconnect: bool = False):
-    with self.dev.acquire() as spi:
-      return self._transfer(spi, 0, struct.pack("<BHHH", request, value, index, 0), timeout, expect_disconnect=expect_disconnect)
+    return self._transfer(0, struct.pack("<BHHH", request, value, index, 0), timeout, expect_disconnect=expect_disconnect)
 
   def controlRead(self, request_type: int, request: int, value: int, index: int, length: int, timeout: int = TIMEOUT):
-    with self.dev.acquire() as spi:
-      return self._transfer(spi, 0, struct.pack("<BHHH", request, value, index, length), timeout)
+    return self._transfer(0, struct.pack("<BHHH", request, value, index, length), timeout, max_rx_len=length)
 
-  # TODO: implement these properly
-  def bulkWrite(self, endpoint: int, data: List[int], timeout: int = TIMEOUT) -> int:
-    with self.dev.acquire() as spi:
-      for x in range(math.ceil(len(data) / XFER_SIZE)):
-        self._transfer(spi, endpoint, data[XFER_SIZE*x:XFER_SIZE*(x+1)], timeout)
-      return len(data)
+  def bulkWrite(self, endpoint: int, data: bytes, timeout: int = TIMEOUT) -> int:
+    for x in range(math.ceil(len(data) / XFER_SIZE)):
+      self._transfer(endpoint, data[XFER_SIZE*x:XFER_SIZE*(x+1)], timeout)
+    return len(data)
 
   def bulkRead(self, endpoint: int, length: int, timeout: int = TIMEOUT) -> bytes:
-    ret: List[int] = []
-    with self.dev.acquire() as spi:
-      for _ in range(math.ceil(length / XFER_SIZE)):
-        d = self._transfer(spi, endpoint, [], timeout, max_rx_len=XFER_SIZE)
-        ret += d
-        if len(d) < XFER_SIZE:
-          break
-    return bytes(ret)
+    ret = b""
+    for _ in range(math.ceil(length / XFER_SIZE)):
+      d = self._transfer(endpoint, [], timeout, max_rx_len=XFER_SIZE)
+      ret += d
+      if len(d) < XFER_SIZE:
+        break
+    return ret
 
 
 class STBootloaderSPIHandle(BaseSTBootloaderHandle):

python/uds.py

@@ -473,7 +473,7 @@ def _isotp_rx_next(self, rx_data: bytes) -> ISOTP_FRAME_TYPE:
     # assert len(rx_data) == self.max_len, f"isotp - rx: invalid CAN frame length: {len(rx_data)}"
 
     if rx_data[0] >> 4 == ISOTP_FRAME_TYPE.SINGLE:
-      self.rx_len = rx_data[0] & 0xFF
+      self.rx_len = rx_data[0] & 0x0F
       assert self.rx_len < self.max_len, f"isotp - rx: invalid single frame length: {self.rx_len}"
       self.rx_dat = rx_data[1:1 + self.rx_len]
       self.rx_idx = 0

python/usb.py

@@ -1,5 +1,4 @@
 import struct
-from typing import List
 
 from .base import BaseHandle, BaseSTBootloaderHandle, TIMEOUT
 from .constants import McuType
@@ -17,7 +16,7 @@ def controlWrite(self, request_type: int, request: int, value: int, index: int,
   def controlRead(self, request_type: int, request: int, value: int, index: int, length: int, timeout: int = TIMEOUT):
     return self._libusb_handle.controlRead(request_type, request, value, index, length, timeout)
 
-  def bulkWrite(self, endpoint: int, data: List[int], timeout: int = TIMEOUT) -> int:
+  def bulkWrite(self, endpoint: int, data: bytes, timeout: int = TIMEOUT) -> int:
     return self._libusb_handle.bulkWrite(endpoint, data, timeout)  # type: ignore
 
   def bulkRead(self, endpoint: int, length: int, timeout: int = TIMEOUT) -> bytes:

tests/hitl/8_spi.py

@@ -45,8 +45,6 @@ def test_protocol_version_data(self, p):
       bstub = v[13]
       assert bstub == (0xEE if bootstub else 0xCC)
 
-      assert v[14:] == b"\x01"
-
   def test_all_comm_types(self, mocker, p):
     spy = mocker.spy(p._handle, '_wait_for_ack')