zfs_debug: Restore log size limit for userspace

For some reason it was dropped when split from kernel, that makes
raidz_test to accumulate in RAM up to 100GB of logs we don't need.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Igor Kozhukhov <igor@dilos.org>
Reviewed-by:  Rob Norris <robn@despairlabs.com>
Reviewed-by: Tino Reichardt <milky-zfs@mcmilk.de>
Signed-off-by:	Alexander Motin <mav@FreeBSD.org>
Sponsored by:	iXsystems, Inc.
Closes openzfs#16492
Closes openzfs#16566
Closes openzfs#16664

lib/libzpool/zfs_debug.c

@@ -35,9 +35,25 @@ typedef struct zfs_dbgmsg {
 
 static list_t zfs_dbgmsgs;
 static kmutex_t zfs_dbgmsgs_lock;
+static uint_t zfs_dbgmsg_size = 0;
+static uint_t zfs_dbgmsg_maxsize = 4<<20; /* 4MB */
 
 int zfs_dbgmsg_enable = B_TRUE;
 
+static void
+zfs_dbgmsg_purge(uint_t max_size)
+{
+	while (zfs_dbgmsg_size > max_size) {
+		zfs_dbgmsg_t *zdm = list_remove_head(&zfs_dbgmsgs);
+		if (zdm == NULL)
+			return;
+
+		uint_t size = zdm->zdm_size;
+		kmem_free(zdm, size);
+		zfs_dbgmsg_size -= size;
+	}
+}
+
 void
 zfs_dbgmsg_init(void)
 {
@@ -74,6 +90,8 @@ __zfs_dbgmsg(char *buf)
 
 	mutex_enter(&zfs_dbgmsgs_lock);
 	list_insert_tail(&zfs_dbgmsgs, zdm);
+	zfs_dbgmsg_size += size;
+	zfs_dbgmsg_purge(zfs_dbgmsg_maxsize);
 	mutex_exit(&zfs_dbgmsgs_lock);
 }