Standardize brake safety tests (commaai#465)

* WIP: improve safety test format

* Fix tests

tests/safety/common.py

@@ -13,26 +13,58 @@ def make_msg(bus, addr, length=8):
 
   return to_send
 
-def test_relay_malfunction(test, addr, bus=0):
-  # input is a test class and the address that, if seen on specified bus, triggers
-  # the relay_malfunction protection logic: both tx_hook and fwd_hook are
-  # expected to return failure
-  test.assertFalse(test.safety.get_relay_malfunction())
-  test.safety.safety_rx_hook(make_msg(bus, addr, 8))
-  test.assertTrue(test.safety.get_relay_malfunction())
-  for a in range(1, 0x800):
-    for b in range(0, 3):
-      test.assertFalse(test.safety.safety_tx_hook(make_msg(b, a, 8)))
-      test.assertEqual(-1, test.safety.safety_fwd_hook(b, make_msg(b, a, 8)))
-
-def test_manually_enable_controls_allowed(test):
-  test.safety.set_controls_allowed(1)
-  test.assertTrue(test.safety.get_controls_allowed())
-  test.safety.set_controls_allowed(0)
-  test.assertFalse(test.safety.get_controls_allowed())
-
-def test_spam_can_buses(test, TX_MSGS):
-  for addr in range(1, 0x800):
-    for bus in range(0, 4):
-      if all(addr != m[0] or bus != m[1] for m in TX_MSGS):
-        test.assertFalse(test.safety.safety_tx_hook(make_msg(bus, addr, 8)))
+class StdTest:
+  @staticmethod
+  def test_relay_malfunction(test, addr, bus=0):
+    # input is a test class and the address that, if seen on specified bus, triggers
+    # the relay_malfunction protection logic: both tx_hook and fwd_hook are
+    # expected to return failure
+    test.assertFalse(test.safety.get_relay_malfunction())
+    test.safety.safety_rx_hook(make_msg(bus, addr, 8))
+    test.assertTrue(test.safety.get_relay_malfunction())
+    for a in range(1, 0x800):
+      for b in range(0, 3):
+        test.assertFalse(test.safety.safety_tx_hook(make_msg(b, a, 8)))
+        test.assertEqual(-1, test.safety.safety_fwd_hook(b, make_msg(b, a, 8)))
+
+  @staticmethod
+  def test_manually_enable_controls_allowed(test):
+    test.safety.set_controls_allowed(1)
+    test.assertTrue(test.safety.get_controls_allowed())
+    test.safety.set_controls_allowed(0)
+    test.assertFalse(test.safety.get_controls_allowed())
+
+  @staticmethod
+  def test_spam_can_buses(test, TX_MSGS):
+    for addr in range(1, 0x800):
+      for bus in range(0, 4):
+        if all(addr != m[0] or bus != m[1] for m in TX_MSGS):
+          test.assertFalse(test.safety.safety_tx_hook(make_msg(bus, addr, 8)))
+
+  @staticmethod
+  def test_allow_brake_at_zero_speed(test):
+    # Brake was already pressed
+    test.safety.safety_rx_hook(test._speed_msg(0))
+    test.safety.safety_rx_hook(test._brake_msg(1))
+    test.safety.set_controls_allowed(1)
+    test.safety.safety_rx_hook(test._brake_msg(1))
+    test.assertTrue(test.safety.get_controls_allowed())
+    test.safety.safety_rx_hook(test._brake_msg(0))
+    test.assertTrue(test.safety.get_controls_allowed())
+    # rising edge of brake should disengage
+    test.safety.safety_rx_hook(test._brake_msg(1))
+    test.assertFalse(test.safety.get_controls_allowed())
+    test.safety.safety_rx_hook(test._brake_msg(0))  # reset no brakes
+
+  @staticmethod
+  def test_not_allow_brake_when_moving(test, standstill_threshold):
+    # Brake was already pressed
+    test.safety.safety_rx_hook(test._brake_msg(1))
+    test.safety.set_controls_allowed(1)
+    test.safety.safety_rx_hook(test._speed_msg(standstill_threshold))
+    test.safety.safety_rx_hook(test._brake_msg(1))
+    test.assertTrue(test.safety.get_controls_allowed())
+    test.safety.safety_rx_hook(test._speed_msg(standstill_threshold + 1))
+    test.safety.safety_rx_hook(test._brake_msg(1))
+    test.assertFalse(test.safety.get_controls_allowed())
+    test.safety.safety_rx_hook(test._speed_msg(0))

tests/safety/test_cadillac.py

@@ -3,7 +3,7 @@
 import numpy as np
 from panda import Panda
 from panda.tests.safety import libpandasafety_py
-from panda.tests.safety.common import make_msg, test_manually_enable_controls_allowed, test_spam_can_buses
+from panda.tests.safety.common import make_msg, StdTest
 
 
 MAX_RATE_UP = 2
@@ -56,13 +56,13 @@ def _torque_msg(self, torque):
     return to_send
 
   def test_spam_can_buses(self):
-    test_spam_can_buses(self, TX_MSGS)
+    StdTest.test_spam_can_buses(self, TX_MSGS)
 
   def test_default_controls_not_allowed(self):
     self.assertFalse(self.safety.get_controls_allowed())
 
   def test_manually_enable_controls_allowed(self):
-    test_manually_enable_controls_allowed(self)
+    StdTest.test_manually_enable_controls_allowed(self)
 
   def test_enable_control_allowed_from_cruise(self):
     to_push = make_msg(0, 0x370)

tests/safety/test_chrysler.py

@@ -3,7 +3,7 @@
 import numpy as np
 from panda import Panda
 from panda.tests.safety import libpandasafety_py
-from panda.tests.safety.common import test_relay_malfunction, make_msg, test_manually_enable_controls_allowed, test_spam_can_buses
+from panda.tests.safety.common import StdTest, make_msg
 
 MAX_RATE_UP = 3
 MAX_RATE_DOWN = 3
@@ -107,10 +107,10 @@ def _torque_msg(self, torque):
     return to_send
 
   def test_spam_can_buses(self):
-    test_spam_can_buses(self, TX_MSGS)
+    StdTest.test_spam_can_buses(self, TX_MSGS)
 
   def test_relay_malfunction(self):
-    test_relay_malfunction(self, 0x292)
+    StdTest.test_relay_malfunction(self, 0x292)
 
   def test_default_controls_not_allowed(self):
     self.assertFalse(self.safety.get_controls_allowed())
@@ -126,7 +126,7 @@ def test_steer_safety_check(self):
           self.assertTrue(self.safety.safety_tx_hook(self._torque_msg(t)))
 
   def test_manually_enable_controls_allowed(self):
-    test_manually_enable_controls_allowed(self)
+    StdTest.test_manually_enable_controls_allowed(self)
 
   def test_enable_control_allowed_from_cruise(self):
     to_push = self._cruise_msg(True)
@@ -149,29 +149,9 @@ def test_gas_disable(self):
     self.safety.safety_rx_hook(self._gas_msg(1))
     self.assertFalse(self.safety.get_controls_allowed())
 
-  def test_brake_disable(self):
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.safety.safety_rx_hook(self._speed_msg(0))
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertTrue(self.safety.get_controls_allowed())
-
-    self.safety.safety_rx_hook(self._brake_msg(False))
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertFalse(self.safety.get_controls_allowed())
-
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.safety.safety_rx_hook(self._speed_msg(1))
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertFalse(self.safety.get_controls_allowed())
-
-    self.safety.safety_rx_hook(self._speed_msg(1))
-    self.safety.safety_rx_hook(self._brake_msg(False))
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertFalse(self.safety.get_controls_allowed())
+  def test_brake_disengage(self):
+    StdTest.test_allow_brake_at_zero_speed(self)
+    StdTest.test_not_allow_brake_when_moving(self, 0)
 
   def test_non_realtime_limit_up(self):
     self.safety.set_controls_allowed(True)

tests/safety/test_gm.py

@@ -3,7 +3,7 @@
 import numpy as np
 from panda import Panda
 from panda.tests.safety import libpandasafety_py
-from panda.tests.safety.common import test_relay_malfunction, make_msg, test_manually_enable_controls_allowed, test_spam_can_buses
+from panda.tests.safety.common import StdTest, make_msg
 
 MAX_RATE_UP = 7
 MAX_RATE_DOWN = 17
@@ -90,10 +90,10 @@ def _torque_msg(self, torque):
     return to_send
 
   def test_spam_can_buses(self):
-    test_spam_can_buses(self, TX_MSGS)
+    StdTest.test_spam_can_buses(self, TX_MSGS)
 
   def test_relay_malfunction(self):
-    test_relay_malfunction(self, 384)
+    StdTest.test_relay_malfunction(self, 384)
 
   def test_default_controls_not_allowed(self):
     self.assertFalse(self.safety.get_controls_allowed())
@@ -116,29 +116,9 @@ def test_cancel_button(self):
     self.safety.safety_rx_hook(self._button_msg(CANCEL_BTN))
     self.assertFalse(self.safety.get_controls_allowed())
 
-  def test_disengage_on_brake(self):
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertFalse(self.safety.get_controls_allowed())
-
-  def test_allow_brake_at_zero_speed(self):
-    # Brake was already pressed
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.safety.set_controls_allowed(1)
-
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertTrue(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._brake_msg(False))
-
-  def test_not_allow_brake_when_moving(self):
-    # Brake was already pressed
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.safety.safety_rx_hook(self._speed_msg(100))
-    self.safety.set_controls_allowed(1)
-
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertFalse(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._brake_msg(False))
+  def test_brake_disengage(self):
+    StdTest.test_allow_brake_at_zero_speed(self)
+    StdTest.test_not_allow_brake_when_moving(self, 0)
 
   def test_disengage_on_gas(self):
     self.safety.set_controls_allowed(1)
@@ -182,7 +162,7 @@ def test_steer_safety_check(self):
           self.assertTrue(self.safety.safety_tx_hook(self._torque_msg(t)))
 
   def test_manually_enable_controls_allowed(self):
-    test_manually_enable_controls_allowed(self)
+    StdTest.test_manually_enable_controls_allowed(self)
 
   def test_non_realtime_limit_up(self):
     self.safety.set_gm_torque_driver(0, 0)

tests/safety/test_honda.py

@@ -3,9 +3,7 @@
 import numpy as np
 from panda import Panda
 from panda.tests.safety import libpandasafety_py
-from panda.tests.safety.common import test_relay_malfunction, make_msg, \
-                                      test_manually_enable_controls_allowed, \
-                                      test_spam_can_buses, MAX_WRONG_COUNTERS
+from panda.tests.safety.common import StdTest, make_msg, MAX_WRONG_COUNTERS
 
 MAX_BRAKE = 255
 
@@ -110,18 +108,18 @@ def test_spam_can_buses(self):
       tx_msgs = BH_TX_MSGS
     elif hw_type == HONDA_BG_HW:
       tx_msgs = BG_TX_MSGS
-    test_spam_can_buses(self, tx_msgs)
+    StdTest.test_spam_can_buses(self, tx_msgs)
 
   def test_relay_malfunction(self):
     hw = self.safety.get_honda_hw()
     bus = 2 if hw == HONDA_BG_HW else 0
-    test_relay_malfunction(self, 0xE4, bus=bus)
+    StdTest.test_relay_malfunction(self, 0xE4, bus=bus)
 
   def test_default_controls_not_allowed(self):
     self.assertFalse(self.safety.get_controls_allowed())
 
   def test_manually_enable_controls_allowed(self):
-    test_manually_enable_controls_allowed(self)
+    StdTest.test_manually_enable_controls_allowed(self)
 
   def test_resume_button(self):
     RESUME_BTN = 4
@@ -167,23 +165,9 @@ def test_alt_disengage_on_brake(self):
     self.safety.safety_rx_hook(self._alt_brake_msg(1))
     self.assertTrue(self.safety.get_controls_allowed())
 
-  def test_allow_brake_at_zero_speed(self):
-    # Brake was already pressed
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.safety.set_controls_allowed(1)
-
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertTrue(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._brake_msg(False))  # reset no brakes
-
-  def test_not_allow_brake_when_moving(self):
-    # Brake was already pressed
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.safety.safety_rx_hook(self._speed_msg(100))
-    self.safety.set_controls_allowed(1)
-
-    self.safety.safety_rx_hook(self._brake_msg(True))
-    self.assertFalse(self.safety.get_controls_allowed())
+  def test_brake_disengage(self):
+    StdTest.test_allow_brake_at_zero_speed(self)
+    StdTest.test_not_allow_brake_when_moving(self, 0)
 
   def test_prev_gas(self):
     self.safety.safety_rx_hook(self._gas_msg(False))

tests/safety/test_hyundai.py

@@ -3,7 +3,7 @@
 import numpy as np
 from panda import Panda
 from panda.tests.safety import libpandasafety_py
-from panda.tests.safety.common import test_relay_malfunction, make_msg, test_manually_enable_controls_allowed, test_spam_can_buses
+from panda.tests.safety.common import StdTest, make_msg
 
 MAX_RATE_UP = 3
 MAX_RATE_DOWN = 7
@@ -74,10 +74,10 @@ def _torque_msg(self, torque):
     return to_send
 
   def test_spam_can_buses(self):
-    test_spam_can_buses(self, TX_MSGS)
+    StdTest.test_spam_can_buses(self, TX_MSGS)
 
   def test_relay_malfunction(self):
-    test_relay_malfunction(self, 832)
+    StdTest.test_relay_malfunction(self, 832)
 
   def test_default_controls_not_allowed(self):
     self.assertFalse(self.safety.get_controls_allowed())
@@ -93,7 +93,7 @@ def test_steer_safety_check(self):
           self.assertTrue(self.safety.safety_tx_hook(self._torque_msg(t)))
 
   def test_manually_enable_controls_allowed(self):
-    test_manually_enable_controls_allowed(self)
+    StdTest.test_manually_enable_controls_allowed(self)
 
   def test_enable_control_allowed_from_cruise(self):
     to_push = make_msg(0, 1057)
@@ -114,30 +114,9 @@ def test_disengage_on_gas(self):
     self.safety.safety_rx_hook(self._gas_msg(1))
     self.assertFalse(self.safety.get_controls_allowed())
 
-  def test_allow_brake_at_zero_speed(self):
-    # Brake was already pressed
-    self.safety.safety_rx_hook(self._brake_msg(1))
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._brake_msg(1))
-    self.assertTrue(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._brake_msg(0))
-    self.assertTrue(self.safety.get_controls_allowed())
-    # rising edge of brake should disengage
-    self.safety.safety_rx_hook(self._brake_msg(1))
-    self.assertFalse(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._brake_msg(0))  # reset no brakes
-
-  def test_not_allow_brake_when_moving(self):
-    # Brake was already pressed
-    self.safety.safety_rx_hook(self._brake_msg(1))
-    self.safety.set_controls_allowed(1)
-    self.safety.safety_rx_hook(self._speed_msg(SPEED_THRESHOLD))
-    self.safety.safety_rx_hook(self._brake_msg(1))
-    self.assertTrue(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._speed_msg(SPEED_THRESHOLD + 1))
-    self.safety.safety_rx_hook(self._brake_msg(1))
-    self.assertFalse(self.safety.get_controls_allowed())
-    self.safety.safety_rx_hook(self._speed_msg(0))
+  def test_brake_disengage(self):
+    StdTest.test_allow_brake_at_zero_speed(self)
+    StdTest.test_not_allow_brake_when_moving(self, SPEED_THRESHOLD)
 
   def test_non_realtime_limit_up(self):
     self.safety.set_hyundai_torque_driver(0, 0)