From 7d953332e0d7c0394c607156bf4099881bdf3f43 Mon Sep 17 00:00:00 2001
From: Michael Stapelberg <stapelberg@users.noreply.github.com>
Date: Sat, 10 Apr 2021 17:03:38 +0200
Subject: [PATCH] ChibiOS USB driver: prevent deadlock with CONSOLE_ENABLE =
 yes (#12472)

Before this commit, attaching an ARM-based (i.e. ChibiOS-based) keyboard that
uses CONSOLE_ENABLE = yes and produces debug messages would deadlock the
keyboard unless one was running hid_listen.

With this commit, dead-locking writes to the queue are detected and prevented.

fixes #5631
---
 tmk_core/protocol/chibios/usb_driver.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tmk_core/protocol/chibios/usb_driver.c b/tmk_core/protocol/chibios/usb_driver.c
index cc0ce7600fc9..eb72f8ff6dc6 100644
--- a/tmk_core/protocol/chibios/usb_driver.c
+++ b/tmk_core/protocol/chibios/usb_driver.c
@@ -80,7 +80,19 @@ static bool qmkusb_start_receive(QMKUSBDriver *qmkusbp) {
  * Interface implementation.
  */
 
-static size_t _write(void *ip, const uint8_t *bp, size_t n) { return obqWriteTimeout(&((QMKUSBDriver *)ip)->obqueue, bp, n, TIME_INFINITE); }
+static size_t _write(void *ip, const uint8_t *bp, size_t n) {
+  output_buffers_queue_t *obqueue = &((QMKUSBDriver *)ip)->obqueue;
+  chSysLock();
+  const bool full = obqIsFullI(obqueue);
+  chSysUnlock();
+  if (full || bqIsSuspendedX(obqueue)) {
+    /* Discard any writes while the queue is suspended or full, i.e. the hidraw
+       interface is not open. If we tried to send with an infinite timeout, we
+       would deadlock the keyboard otherwise. */
+    return -1;
+  }
+  return obqWriteTimeout(obqueue, bp, n, TIME_INFINITE);
+}
 
 static size_t _read(void *ip, uint8_t *bp, size_t n) { return ibqReadTimeout(&((QMKUSBDriver *)ip)->ibqueue, bp, n, TIME_INFINITE); }