Merge pull request #1375 from tempesta-tech/ak-bn-mem-profiles

#1064: TLS performance imporovements

Makefile

@@ -1,7 +1,7 @@
 #		Tempesta FW
 #
 # Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
-# Copyright (C) 2015-2019 Tempesta Technologies, Inc.
+# Copyright (C) 2015-2020 Tempesta Technologies, Inc.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -56,19 +56,31 @@ endif
 ifeq (, $(findstring sse4_2, $(PROC)))
 	ERROR = "SSE 4.2 support is required"
 endif
+ifeq (, $(findstring pse, $(PROC)))
+	ERROR = "1MB huge pages support is required"
+endif
 ifneq (, $(findstring avx2, $(PROC)))
 	AVX2 = "y"
 	TFW_CFLAGS += -DAVX2=1
 endif
-ifeq (, $(findstring pse, $(PROC)))
-	ERROR = "1MB huge pages support is required"
+ifneq (, $(findstring bmi2, $(PROC)))
+	BMI2 = "y"
+	TFW_CFLAGS += -DBMI2=1
+else
+	ERROR = "BMI2 CPU extension is required for Tempesta TLS"
+endif
+ifneq (, $(findstring adx, $(PROC)))
+	ADX = "y"
+	TFW_CFLAGS += -DADX=1
+else
+	ERROR = "ADX CPU extension is required for Tempesta TLS"
 endif
 
 TFW_CFLAGS += -mmmx -msse4.2
 
 KERNEL = /lib/modules/$(shell uname -r)/build
 
-export KERNEL TFW_CFLAGS AVX2 TFW_GCOV
+export KERNEL TFW_CFLAGS AVX2 BMI2 ADX TFW_GCOV
 
 obj-m	+= lib/ tempesta_db/core/ tempesta_fw/ tls/
 
@@ -79,7 +91,11 @@ ifdef ERROR
 	$(error $(ERROR))
 endif
 ifndef AVX2
-	$(warning WARNING: NO AVX2 SUPPORT, YOU WILL BE SLOW)
+	$(warning !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!)
+	$(warning WARNING: YOUR PLATFORM IS TOO OLD AND IS NOT UNSUPPORTED)
+	$(warning WARNING: THIS AFFECT PERFORMANCE AND MIGHT AFFECT SECURITY)
+	$(warning WARNING: PLEASE DO NOT USE THE BUILD IN PRODUCTION)
+	$(warning !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!)
 endif
 	$(MAKE) -C tempesta_db
 	$(MAKE) -C $(KERNEL) M=$(shell pwd) modules

ktest/README.md

@@ -1,8 +1,18 @@
-## Tempesta Kernel Emulation Unit Testing Framework
+# Tempesta Kernel Emulation Framework for Unit Testing
 
-This is just a set of C headers with the same names and directory hierarchies
-as in Linux kernel plus some additional helpers. This headers provide only most
-usable stubs for real kernel headers, so small and kernel modules can be
-compiled and tested in user space.
+Light-weigh Linux kernel mocking headers-only library for unit testing.
+Read description and comparison with other testing framework at the
+[Wiki](https://github.com/tempesta-tech/tempesta/wiki/Testing).
 
-See usage example in tmepesta\_db/t/tdb\_htrie.c.
+See usage examples in tempesta\_db/t/tdb\_htrie.c and tls/t/test\_tls.c.
+
+
+# Multi-processing
+
+The framework emulates 32 CPUs by default using pthreads and arrays for
+per-cpu variables. To use it for single-threaded testing, redefine `NR_CPUS`
+before the file include:
+```C
+#define NR_CPUS 1
+#include "ktest.h"
+```

ktest/asm/alternative-asm.h

@@ -0,0 +1,23 @@
+/**
+ *	Tempesta kernel emulation unit testing framework.
+ *
+ * Copyright (C) 2020 Tempesta Technologies, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __ALTERNATIVE_ASM_H__
+#define __ALTERNATIVE_ASM_H__
+
+#endif /* __ALTERNATIVE_ASM_H__ */

ktest/asm/export.h

@@ -0,0 +1,23 @@
+/**
+ *	Tempesta kernel emulation unit testing framework.
+ *
+ * Copyright (C) 2020 Tempesta Technologies, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __EXPORT_H__
+#define __EXPORT_H__
+
+#endif /* __EXPORT_H__ */

ktest/asm/fpu/api.h

@@ -0,0 +1,27 @@
+/**
+ *	Tempesta kernel emulation unit testing framework.
+ *
+ * Copyright (C) 2019 Tempesta Technologies, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __FPU_API_H__
+#define __FPU_API_H__
+
+/* SIMD operations are safe in user space. */
+#define kernel_fpu_begin()
+#define kernel_fpu_end()
+
+#endif /* __FPI_API_H__ */

ktest/asm/nospec-branch.h

@@ -0,0 +1,25 @@
+/**
+ *	Tempesta kernel emulation unit testing framework.
+ *
+ * Copyright (C) 2020 Tempesta Technologies, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __NOSPEC_BRANCH_H__
+#define __NOSPEC_BRANCH_H__
+
+#define ANNOTATE_RETPOLINE_SAFE
+
+#endif /* __NOSPEC_BRANCH_H__ */

ktest/asm/sync_bitops.h

@@ -1,7 +1,7 @@
 /**
  *	Tempesta kernel emulation unit testing framework.
  *
- * Copyright (C) 2015 Tempesta Technologies.
+ * Copyright (C) 2015-2019 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -33,4 +33,23 @@ sync_test_and_set_bit(int nr, volatile unsigned long *addr)
 	return oldbit;
 }
 
+static inline unsigned long
+__ffs(unsigned long word)
+{
+	asm("rep; bsf %1,%0"
+		: "=r" (word)
+		: "rm" (word));
+	return word;
+}
+
+static inline int
+fls64(__u64 x)
+{
+	int bitpos = -1;
+	asm("bsrq %1,%q0"
+	    : "+r" (bitpos)
+	    : "rm" (x));
+	return bitpos + 1;
+}
+
 #endif /* __SYNC_BITOPS_H__ */

ktest/crypto/hash.h

@@ -0,0 +1,96 @@
+/**
+ *	Tempesta kernel emulation unit testing framework.
+ *
+ * Copyright (C) 2019-2020 Tempesta Technologies, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __CRYPTO_HASH_H__
+#define __CRYPTO_HASH_H__
+
+#include "linux/kernel.h"
+#include "linux/atomic.h"
+
+#define CRYPTO_MAX_ALG_NAME	128
+
+struct crypto_shash { /* dummy struct */ };
+struct shash_desc { /* dummy struct */ };
+struct crypto_tfm { /* dummy struct */ };
+struct crypto_type { /* dummy struct */ };
+struct ablkcipher_alg { /* dummy struct */ };
+struct blkcipher_alg { /* dummy struct */ };
+struct cipher_alg { /* dummy struct */ };
+struct compress_alg { /* dummy struct */ };
+
+struct crypto_alg {
+	struct list_head cra_list;
+	struct list_head cra_users;
+
+	u32 cra_flags;
+	unsigned int cra_blocksize;
+	unsigned int cra_ctxsize;
+	unsigned int cra_alignmask;
+
+	int cra_priority;
+	atomic_t cra_refcnt;
+
+	char cra_name[CRYPTO_MAX_ALG_NAME];
+	char cra_driver_name[CRYPTO_MAX_ALG_NAME];
+
+	const struct crypto_type *cra_type;
+
+	union {
+		struct ablkcipher_alg ablkcipher;
+		struct blkcipher_alg blkcipher;
+		struct cipher_alg cipher;
+		struct compress_alg compress;
+	} cra_u;
+
+	int (*cra_init)(struct crypto_tfm *tfm);
+	void (*cra_exit)(struct crypto_tfm *tfm);
+	void (*cra_destroy)(struct crypto_alg *alg);
+
+	struct module *cra_module;
+} CRYPTO_MINALIGN_ATTR;
+
+struct shash_alg {
+	int (*init)(struct shash_desc *desc);
+	int (*update)(struct shash_desc *desc, const u8 *data,
+		      unsigned int len);
+	int (*final)(struct shash_desc *desc, u8 *out);
+	int (*finup)(struct shash_desc *desc, const u8 *data,
+		     unsigned int len, u8 *out);
+	int (*digest)(struct shash_desc *desc, const u8 *data,
+		      unsigned int len, u8 *out);
+	int (*export)(struct shash_desc *desc, void *out);
+	int (*import)(struct shash_desc *desc, const void *in);
+	int (*setkey)(struct crypto_shash *tfm, const u8 *key,
+		      unsigned int keylen);
+
+	unsigned int descsize;
+
+	/* These fields must match hash_alg_common. */
+	unsigned int digestsize CRYPTO_MINALIGN_ATTR;
+	unsigned int statesize;
+
+	struct crypto_alg base;
+};
+
+static inline struct shash_alg *__crypto_shash_alg(struct crypto_alg *alg)
+{
+	return container_of(alg, struct shash_alg, base);
+}
+
+#endif /* __CRYPTO_HASH_H__ */

ktest/crypto/sha.h

@@ -0,0 +1,43 @@
+/**
+ *	Tempesta kernel emulation unit testing framework.
+ *
+ * Copyright (C) 2019 Tempesta Technologies, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+#ifndef __CRYPTO_SHA_H__
+#define __CRYPTO_SHA_H__
+
+#include "linux/compiler.h"
+
+#define SHA512_DIGEST_SIZE	64
+#define SHA512_BLOCK_SIZE	128
+
+#define SHA256_DIGEST_SIZE	32
+#define SHA256_BLOCK_SIZE	64
+
+struct sha512_state {
+	u64 state[SHA512_DIGEST_SIZE / 8];
+	u64 count[2];
+	u8 buf[SHA512_BLOCK_SIZE];
+};
+
+struct sha256_state {
+	u32 state[SHA256_DIGEST_SIZE / 4];
+	u64 count[2];
+	u8 buf[SHA256_BLOCK_SIZE];
+};
+
+#endif /* __CRYPTO_SHA_H__ */