Make use of kvmalloc if available

This patch implements use of kvmalloc for GFP_KERNEL allocations, which
may increase performance if the allocator is able to allocate physical
memory, if kvmalloc is available as a public kernel interface (since
v4.12). Otherwise it will simply fall back to virtual memory (vmalloc).

Signed-off-by: Sebastian Gottschall <s.gottschall@dd-wrt.com>
Signed-off-by: Michael Niewöhner <foss@mniewoehner.de>

config/kernel-kmem.m4

@@ -56,3 +56,24 @@ AC_DEFUN([SPL_AC_DEBUG_KMEM_TRACKING], [
 	AC_MSG_CHECKING([whether detailed kmem tracking is enabled])
 	AC_MSG_RESULT([$enable_debug_kmem_tracking])
 ])
+
+dnl #
+dnl # 4.12 API
+dnl #
+dnl # kvmalloc allocation strategy
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_KVMALLOC], [
+	AC_MSG_CHECKING([whether kvmalloc(ptr, flags) is available])
+	ZFS_LINUX_TRY_COMPILE([
+		#include <linux/mm.h>
+	],[
+		void *p __attribute__ ((unused));
+
+		p = kvmalloc(0, GFP_KERNEL);
+	],[
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_KVMALLOC, 1, [kvmalloc exists])
+	],[
+		AC_MSG_RESULT([no])
+	])
+])

config/kernel.m4

@@ -25,6 +25,7 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
 	ZFS_AC_KERNEL_3ARGS_VFS_GETATTR
 	ZFS_AC_KERNEL_2ARGS_VFS_GETATTR
 	ZFS_AC_KERNEL_USLEEP_RANGE
+	ZFS_AC_KERNEL_KVMALLOC
 	ZFS_AC_KERNEL_KMEM_CACHE_ALLOCFLAGS
 	ZFS_AC_KERNEL_KMEM_CACHE_CREATE_USERCOPY
 	ZFS_AC_KERNEL_WAIT_ON_BIT

module/spl/spl-kmem-cache.c

@@ -203,7 +203,7 @@ kv_alloc(spl_kmem_cache_t *skc, int size, int flags)
 		ASSERT(ISP2(size));
 		ptr = (void *)__get_free_pages(lflags, get_order(size));
 	} else {
-		ptr = __vmalloc(size, lflags | __GFP_HIGHMEM, PAGE_KERNEL);
+		ptr = spl_kmem_alloc_impl(size, lflags, NUMA_NO_NODE);
 	}
 
 	/* Resulting allocated memory will be page aligned */
@@ -231,7 +231,7 @@ kv_free(spl_kmem_cache_t *skc, void *ptr, int size)
 		ASSERT(ISP2(size));
 		free_pages((unsigned long)ptr, get_order(size));
 	} else {
-		vfree(ptr);
+		spl_kmem_free_impl(ptr, size);
 	}
 }
 

module/spl/spl-kmem.c

@@ -27,6 +27,7 @@
 #include <sys/kmem.h>
 #include <sys/vmem.h>
 #include <linux/mm.h>
+#include <linux/vmalloc.h>
 
 /*
  * As a general rule kmem_alloc() allocations should be small, preferably
@@ -177,16 +178,34 @@ spl_kmem_alloc_impl(size_t size, int flags, int node)
 		 * acquired.  Contention on this lock can significantly
 		 * impact performance so frequently manipulating the virtual
 		 * address space is strongly discouraged.
+		 *
+		 * GFP_KERNEL allocations can safely use kvmalloc which may
+		 * improve performance by avoiding a) high latency caused by
+		 * vmalloc's on-access allocation, b) performance loss due to
+		 * MMU memory address mapping and c) vmalloc locking overhead.
+		 * This has the side-effect that the slab statistics will
+		 * incorrectly report this as a vmem allocation, but that is
+		 * purely cosmetic.
+		 *
+		 * However, kvmalloc will only be used when a) sufficient
+		 * contiguous memory is available and b) the allocated memory
+		 * is less than spl_kmem_alloc_max.
 		 */
-		if ((size > spl_kmem_alloc_max) || use_vmem) {
+		if ((size > spl_kmem_alloc_max) || use_vmem ||
+		    !((flags & GFP_KERNEL) != GFP_KERNEL)) {
 			if (flags & KM_VMEM) {
 				ptr = __vmalloc(size, lflags | __GFP_HIGHMEM,
 				    PAGE_KERNEL);
 			} else {
 				return (NULL);
 			}
 		} else {
-			ptr = kmalloc_node(size, lflags, node);
+#ifdef HAVE_KVMALLOC
+			if (flags & KM_VMEM) {
+				return (kvmalloc(size, flags));
+			} else
+#endif /* HAVE_KVMALLOC */
+				ptr = kmalloc_node(size, lflags, node);
 		}
 
 		if (likely(ptr) || (flags & KM_NOSLEEP))