modprobe iomemory-vsl4 hangs on Kernel 5.3.18 with USE_QUEUE_MQ

[Tourman36]

hangs on load and modprobe cannot be killed

root@pve1:~# strace -rf modprobe iomemory-vsl4 0.000000 execve("/sbin/modprobe", ["modprobe", "iomemory-vsl4"], 0x7ffe33b4c870 /* 19 vars */) = 0 0.000908 brk(NULL) = 0x55cf0e499000 0.000142 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) 0.000375 openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 0.000128 fstat(3, {st_mode=S_IFREG|0644, st_size=62689, ...}) = 0 0.000216 mmap(NULL, 62689, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6291500000 0.000097 close(3) = 0 0.000100 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3 0.000082 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0205\0\0\0\0\0\0"..., 832) = 832 0.000129 fstat(3, {st_mode=S_IFREG|0644, st_size=158400, ...}) = 0 0.000069 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f62914fe000 0.000082 mmap(NULL, 160400, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f62914d6000 0.000066 mmap(0x7f62914d9000, 98304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f62914d9000 0.000077 mmap(0x7f62914f1000, 45056, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x7f62914f1000 0.000079 mmap(0x7f62914fc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7f62914fc000 0.000107 close(3) = 0 0.000080 openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = 3 0.000087 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\360\10\0\0\0\0\0"..., 832) = 832 0.000092 fstat(3, {st_mode=S_IFREG|0644, st_size=3031904, ...}) = 0 0.000079 mmap(NULL, 3051424, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f62911ed000 0.000077 mprotect(0x7f6291272000, 2285568, PROT_NONE) = 0 0.000081 mmap(0x7f6291272000, 1695744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x85000) = 0x7f6291272000 0.000086 mmap(0x7f6291410000, 585728, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x223000) = 0x7f6291410000 0.000082 mmap(0x7f62914a0000, 204800, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2b2000) = 0x7f62914a0000 0.000115 mmap(0x7f62914d2000, 16288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f62914d2000 0.000099 close(3) = 0 0.000095 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 0.000089 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260A\2\0\0\0\0\0"..., 832) = 832 0.000076 fstat(3, {st_mode=S_IFREG|0755, st_size=1824496, ...}) = 0 0.000075 mmap(NULL, 1837056, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f629102c000 0.000078 mprotect(0x7f629104e000, 1658880, PROT_NONE) = 0 0.000074 mmap(0x7f629104e000, 1343488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f629104e000 0.000077 mmap(0x7f6291196000, 311296, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16a000) = 0x7f6291196000 0.000073 mmap(0x7f62911e3000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x7f62911e3000 0.000089 mmap(0x7f62911e9000, 14336, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f62911e9000 0.000096 close(3) = 0 0.000082 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 0.000085 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@l\0\0\0\0\0\0"..., 832) = 832 0.000075 fstat(3, {st_mode=S_IFREG|0755, st_size=146968, ...}) = 0 0.000080 mmap(NULL, 132288, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f629100b000 0.000082 mmap(0x7f6291011000, 61440, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f6291011000 0.000078 mmap(0x7f6291020000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f6291020000 0.000073 mmap(0x7f6291026000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7f6291026000 0.000090 mmap(0x7f6291028000, 13504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f6291028000 0.000087 close(3) = 0 0.000070 openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 0.000083 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\21\0\0\0\0\0\0"..., 832) = 832 0.000071 fstat(3, {st_mode=S_IFREG|0644, st_size=14592, ...}) = 0 0.000076 mmap(NULL, 16656, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f6291006000 0.000071 mmap(0x7f6291007000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f6291007000 0.000079 mmap(0x7f6291008000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f6291008000 0.000071 mmap(0x7f6291009000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f6291009000 0.000091 close(3) = 0 0.000089 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6291004000 0.000085 arch_prctl(ARCH_SET_FS, 0x7f6291005480) = 0 0.000229 mprotect(0x7f62911e3000, 16384, PROT_READ) = 0 0.000085 mprotect(0x7f6291009000, 4096, PROT_READ) = 0 0.000073 mprotect(0x7f6291026000, 4096, PROT_READ) = 0 0.001580 mprotect(0x7f62914a0000, 196608, PROT_READ) = 0 0.000081 mprotect(0x7f62914fc000, 4096, PROT_READ) = 0 0.000163 mprotect(0x55cf0dd0e000, 8192, PROT_READ) = 0 0.000076 mprotect(0x7f6291537000, 4096, PROT_READ) = 0 0.000066 munmap(0x7f6291500000, 62689) = 0 0.000092 set_tid_address(0x7f6291005750) = 3040392 0.000097 set_robust_list(0x7f6291005760, 24) = 0 0.000065 rt_sigaction(SIGRTMIN, {sa_handler=0x7f62910116b0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f629101d730}, NULL, 8) = 0 0.000119 rt_sigaction(SIGRT_1, {sa_handler=0x7f6291011740, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f629101d730}, NULL, 8) = 0 0.000075 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 0.000078 prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0 0.000320 brk(NULL) = 0x55cf0e499000 0.000060 brk(0x55cf0e4ba000) = 0x55cf0e4ba000 0.000083 uname({sysname="Linux", nodename="pve1", ...}) = 0 0.000088 stat("/etc/modprobe.d", {st_mode=S_IFDIR|0755, st_size=5, ...}) = 0 0.000095 openat(AT_FDCWD, "/etc/modprobe.d", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3 0.000075 fstat(3, {st_mode=S_IFDIR|0755, st_size=5, ...}) = 0 0.000077 getdents64(3, /* 5 entries */, 32768) = 160 0.000152 newfstatat(3, "pve-blacklist.conf", {st_mode=S_IFREG|0644, st_size=171, ...}, 0) = 0 0.000092 newfstatat(3, "iomemory-vsl4.conf", {st_mode=S_IFREG|0644, st_size=436, ...}, 0) = 0 0.000093 newfstatat(3, "dkms.conf", {st_mode=S_IFREG|0644, st_size=127, ...}, 0) = 0 0.000086 getdents64(3, /* 0 entries */, 32768) = 0 0.000101 close(3) = 0 0.000068 stat("/run/modprobe.d", 0x7fffd46bebd0) = -1 ENOENT (No such file or directory) 0.000099 stat("/lib/modprobe.d", {st_mode=S_IFDIR|0755, st_size=8, ...}) = 0 0.000090 openat(AT_FDCWD, "/lib/modprobe.d", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3 0.000076 fstat(3, {st_mode=S_IFDIR|0755, st_size=8, ...}) = 0 0.000085 getdents64(3, /* 8 entries */, 32768) = 344 0.000102 newfstatat(3, "blacklist_pve-kernel-4.15.18-12-pve.conf", {st_mode=S_IFREG|0644, st_size=1239, ...}, 0) = 0 0.000089 newfstatat(3, "aliases.conf", {st_mode=S_IFREG|0644, st_size=655, ...}, 0) = 0 0.000086 newfstatat(3, "blacklist_pve-kernel-5.3.18-3-pve.conf", {st_mode=S_IFREG|0644, st_size=1321, ...}, 0) = 0 0.000090 newfstatat(3, "systemd.conf", {st_mode=S_IFREG|0644, st_size=765, ...}, 0) = 0 0.000084 newfstatat(3, "blacklist_pve-kernel-4.15.18-27-pve.conf", {st_mode=S_IFREG|0644, st_size=1239, ...}, 0) = 0 0.000091 newfstatat(3, "fbdev-blacklist.conf", {st_mode=S_IFREG|0644, st_size=390, ...}, 0) = 0 0.000085 getdents64(3, /* 0 entries */, 32768) = 0 0.000075 close(3) = 0 0.000076 openat(AT_FDCWD, "/lib/modprobe.d/aliases.conf", O_RDONLY|O_CLOEXEC) = 3 0.000117 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000083 fstat(3, {st_mode=S_IFREG|0644, st_size=655, ...}) = 0 0.000075 read(3, "# These are the standard aliases"..., 1024) = 655 0.000133 read(3, "", 1024) = 0 0.000072 close(3) = 0 0.000070 openat(AT_FDCWD, "/lib/modprobe.d/blacklist_pve-kernel-4.15.18-12-pve.conf", O_RDONLY|O_CLOEXEC) = 3 0.000083 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000069 fstat(3, {st_mode=S_IFREG|0644, st_size=1239, ...}) = 0 0.000072 read(3, "blacklist acquirewdt\nblacklist a"..., 1536) = 1239 0.000132 read(3, "", 1536) = 0 0.000064 close(3) = 0 0.000063 openat(AT_FDCWD, "/lib/modprobe.d/blacklist_pve-kernel-4.15.18-27-pve.conf", O_RDONLY|O_CLOEXEC) = 3 0.000073 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000065 fstat(3, {st_mode=S_IFREG|0644, st_size=1239, ...}) = 0 0.000067 read(3, "blacklist acquirewdt\nblacklist a"..., 1536) = 1239 0.000129 read(3, "", 1536) = 0 0.000063 close(3) = 0 0.000062 openat(AT_FDCWD, "/lib/modprobe.d/blacklist_pve-kernel-5.3.18-3-pve.conf", O_RDONLY|O_CLOEXEC) = 3 0.000082 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000066 fstat(3, {st_mode=S_IFREG|0644, st_size=1321, ...}) = 0 0.000066 read(3, "blacklist acquirewdt\nblacklist a"..., 1536) = 1321 0.000145 read(3, "", 1536) = 0 0.000063 close(3) = 0 0.000062 openat(AT_FDCWD, "/etc/modprobe.d/dkms.conf", O_RDONLY|O_CLOEXEC) = 3 0.000072 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000060 fstat(3, {st_mode=S_IFREG|0644, st_size=127, ...}) = 0 0.000067 read(3, "# modprobe information used for "..., 512) = 127 0.000085 read(3, "", 512) = 0 0.000063 close(3) = 0 0.000061 openat(AT_FDCWD, "/lib/modprobe.d/fbdev-blacklist.conf", O_RDONLY|O_CLOEXEC) = 3 0.000071 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000059 fstat(3, {st_mode=S_IFREG|0644, st_size=390, ...}) = 0 0.000066 read(3, "# This file blacklists most old-"..., 512) = 390 0.000102 read(3, "", 512) = 0 0.000063 close(3) = 0 0.000062 openat(AT_FDCWD, "/etc/modprobe.d/iomemory-vsl4.conf", O_RDONLY|O_CLOEXEC) = 3 0.000072 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000060 fstat(3, {st_mode=S_IFREG|0644, st_size=436, ...}) = 0 0.000065 read(3, "# To keep ioDrive from auto load"..., 512) = 436 0.000082 read(3, "", 512) = 0 0.000064 close(3) = 0 0.000063 openat(AT_FDCWD, "/lib/modules/5.3.18-3-pve/modules.softdep", O_RDONLY|O_CLOEXEC) = 3 0.000076 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000061 fstat(3, {st_mode=S_IFREG|0644, st_size=830, ...}) = 0 0.000070 read(3, "# Soft dependencies extracted fr"..., 1024) = 830 0.000124 read(3, "", 1024) = 0 0.000062 close(3) = 0 0.000061 openat(AT_FDCWD, "/etc/modprobe.d/pve-blacklist.conf", O_RDONLY|O_CLOEXEC) = 3 0.000071 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000060 fstat(3, {st_mode=S_IFREG|0644, st_size=171, ...}) = 0 0.000065 read(3, "# This file contains a list of m"..., 512) = 171 0.000080 read(3, "", 512) = 0 0.000078 close(3) = 0 0.000069 openat(AT_FDCWD, "/lib/modprobe.d/systemd.conf", O_RDONLY|O_CLOEXEC) = 3 0.000076 fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE) 0.000061 fstat(3, {st_mode=S_IFREG|0644, st_size=765, ...}) = 0 0.000066 read(3, "# SPDX-License-Identifier: LGPL"..., 1024) = 765 0.000089 read(3, "", 1024) = 0 0.000062 close(3) = 0 0.000060 openat(AT_FDCWD, "/proc/cmdline", O_RDONLY|O_CLOEXEC) = 3 0.000081 read(3, "BOOT_IMAGE=/ROOT/pve-1@/boot/vml"..., 4095) = 120 0.000075 read(3, "", 3975) = 0 0.000059 close(3) = 0 0.000084 openat(AT_FDCWD, "/lib/modules/5.3.18-3-pve/modules.dep.bin", O_RDONLY|O_CLOEXEC) = 3 0.000077 fstat(3, {st_mode=S_IFREG|0644, st_size=850246, ...}) = 0 0.000068 mmap(NULL, 850246, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6290f34000 0.000084 close(3) = 0 0.000068 openat(AT_FDCWD, "/lib/modules/5.3.18-3-pve/modules.alias.bin", O_RDONLY|O_CLOEXEC) = 3 0.000083 fstat(3, {st_mode=S_IFREG|0644, st_size=1347637, ...}) = 0 0.000076 mmap(NULL, 1347637, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6290dea000 0.000087 close(3) = 0 0.000065 openat(AT_FDCWD, "/lib/modules/5.3.18-3-pve/modules.symbols.bin", O_RDONLY|O_CLOEXEC) = 3 0.000084 fstat(3, {st_mode=S_IFREG|0644, st_size=722478, ...}) = 0 0.000075 mmap(NULL, 722478, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6290d39000 0.000084 close(3) = 0 0.000065 openat(AT_FDCWD, "/lib/modules/5.3.18-3-pve/modules.builtin.bin", O_RDONLY|O_CLOEXEC) = 3 0.000084 fstat(3, {st_mode=S_IFREG|0644, st_size=10399, ...}) = 0 0.000080 mmap(NULL, 10399, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f629150d000 0.000083 close(3) = 0 0.000127 openat(AT_FDCWD, "/sys/module/iomemory_vsl4/initstate", O_RDONLY|O_CLOEXEC) = 3 0.000095 read(3, "coming\n", 31) = 7 0.000079 read(3, "", 24) = 0 0.000069 close(3) = 0 0.000080 openat(AT_FDCWD, "/sys/module/iomemory_vsl4/initstate", O_RDONLY|O_CLOEXEC) = 3 0.000079 read(3, "coming\n", 31) = 7 0.000072 read(3, "", 24) = 0 0.000067 close(3) = 0 0.000082 openat(AT_FDCWD, "/lib/modules/5.3.18-3-pve/extra/fio/iomemory-vsl4.ko", O_RDONLY|O_CLOEXEC) = 3 0.000087 read(3, "\177ELF\2\1", 6) = 6 0.000097 lseek(3, 0, SEEK_SET) = 0 0.000080 fstat(3, {st_mode=S_IFREG|0644, st_size=6336176, ...}) = 0 0.000079 mmap(NULL, 6336176, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f629072e000 0.000083 finit_module(3, "external_power_override=1443G108"..., 0

[snuf]

@Tourman36 I assume this is the code-cleanup branch ? If so it's the file_ops vs proc_ops change that took place, I'll have a look later tonight.

[Tourman36]

@snuf nah this was on my branch. I am not sure what causes it to hang, but it hosed my dev box. I need to rebuild it and test on a VM.

[snuf]

@Tourman36 I'll check the code-cleanup branch tonight just to make sure, as I wasn't able to check it yesterday. Make sure you note which branch it is as it will confuse people with regard to the state of things.

[Tourman36]

@Tourman36 I'll check the code-cleanup branch tonight just to make sure, as I wasn't able to check it yesterday. Make sure you note which branch it is as it will confuse people with regard to the state of things.

It doesn't let me select a project or branch here. If you can do it, please go ahead.

[snuf]

@Tourman36 this was fixed right with disabling USE_QUEUE_MQ?

[Tourman36]

@snuf yeah disabling USE_QUEUE_MQ fixes it. Per our discussion this needs additional work to fully implement.

[Tourman36]

I fixed a few null pointer dereference errors, but we're still getting hard crashes or system lockups. Last one I was able to catch:

Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.654163] ------------[ cut here ]------------
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.654165] kernel BUG at block/blk-softirq.c:105!
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.655072] invalid opcode: 0000 [#1] SMP PTI
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.655807] CPU: 1 PID: 36422 Comm: fio-wq/1 Tainted: G OE 5.4.0-24-generic #28-Ubuntu
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.656544] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 0.0.0 02/06/2015
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.657334] RIP: 0010:__blk_complete_request+0xf9/0x100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.658108] Code: c7 83 08 01 00 00 00 00 00 00 e8 d2 13 c6 ff eb 9f 44 89 ee 44 89 f7 e8 75 dc be ff eb b0 bf 04 00 00 00 e8 99 aa bb ff eb 86 <0f> 0b 90 90 90 90 90 0f 1f 44 00 00 48 8b 05 64 79 11 01 55 48 89
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.659676] RSP: 0018:ffffb278825afca8 EFLAGS: 00010246
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.660433] RAX: ffffffffc08ee780 RBX: ffff8c14fd1c5230 RCX: ffffffffc07f2ee0
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.661220] RDX: ffff8c14fd1c5230 RSI: 0000000000001000 RDI: ffff8c14fd1e2100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.662003] RBP: ffffb278825afcc8 R08: ffffffffc07f2870 R09: 0000000000000201
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.662777] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8c14fd1e2100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.663506] R13: 0000000000000001 R14: ffff8c14fbb6c280 R15: 0000000000000003
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.664243] FS: 0000000000000000(0000) GS:ffff8c15fba80000(0000) knlGS:0000000000000000
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.664968] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.665733] CR2: 00007f0b65a828e0 CR3: 0000000131b62000 CR4: 00000000000006e0
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.666466] Call Trace:
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.667148] blk_mq_complete_request+0xbc/0x100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.667901] kfio_req_completor+0x1d/0x50 [iomemory_vsl4]
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.668581] ifio_f9142.4bb664afe4728d2c3817c99088f2b5dd004.4.3.7.1205+0xa1/0x450 [iomemory_vsl4]
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.669300] ifio_015db.f4472ed7df1b82d80c0d8d7eb68abe47d24.4.3.7.1205+0x90e/0x11d0 [iomemory_vsl4]
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.669979] ? del_timer_sync+0x29/0x40
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.670691] ? schedule_timeout+0x15a/0x2f0
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.671362] ? __next_timer_interrupt+0xe0/0xe0
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.672056] ifio_1e5a3.5e62afc90b8d1b25ae145e583e480994677.4.3.7.1205+0x27f/0x18d0 [iomemory_vsl4]
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.672775] ifio_23a88.d728a86e123ff2fa33650aa276c25af938b.4.3.7.1205+0x578/0x920 [iomemory_vsl4]
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.673536] kthread+0x104/0x140
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.674263] ? ifio_23a88.d728a86e123ff2fa33650aa276c25af938b.4.3.7.1205+0x310/0x920 [iomemory_vsl4]
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.675038] ? kthread_park+0x90/0x90
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.675770] ret_from_fork+0x35/0x40
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.676491] Modules linked in: iomemory_vsl4(OE+) nls_iso8859_1 dm_multipath scsi_dh_rdac scsi_dh_emc scsi_dh_alua joydev input_leds serio_raw mac_hid qemu_fw_cfg sch_fq_codel ip_tables x_tables autofs4 btrfs zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear hid_generic usbhid hid bochs_drm drm_vram_helper ttm drm_kms_helper psmouse virtio_net syscopyarea net_failover sysfillrect sysimgblt fb_sys_fops failover drm virtio_blk i2c_piix4 pata_acpi floppy
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.680816] ---[ end trace 5f210f14e6cad8ce ]---
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.682103] RIP: 0010:__blk_complete_request+0xf9/0x100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.683012] Code: c7 83 08 01 00 00 00 00 00 00 e8 d2 13 c6 ff eb 9f 44 89 ee 44 89 f7 e8 75 dc be ff eb b0 bf 04 00 00 00 e8 99 aa bb ff eb 86 <0f> 0b 90 90 90 90 90 0f 1f 44 00 00 48 8b 05 64 79 11 01 55 48 89
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.684770] RSP: 0018:ffffb278825afca8 EFLAGS: 00010246
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.685749] RAX: ffffffffc08ee780 RBX: ffff8c14fd1c5230 RCX: ffffffffc07f2ee0
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.686813] RDX: ffff8c14fd1c5230 RSI: 0000000000001000 RDI: ffff8c14fd1e2100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.687720] RBP: ffffb278825afcc8 R08: ffffffffc07f2870 R09: 0000000000000201
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.688629] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8c14fd1e2100
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.689620] R13: 0000000000000001 R14: ffff8c14fbb6c280 R15: 0000000000000003
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.690674] FS: 0000000000000000(0000) GS:ffff8c15fba80000(0000) knlGS:0000000000000000
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.691574] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
Apr 19 13:43:40 ubuntu-dev kernel: [ 1105.692461] CR2: 00007f0b65a828e0 CR3: 0000000131b62000 CR4: 00000000000006e0

[Tourman36]

Fixed modprobe hang, but code is still unstable and causes the system to lockup. Probably memory starvation. Need to investigate further.