Merge pull request #646 from devimc/topic/kernel/fragmentsSupport

Kernel:  add config fragment support

kernel/build-kernel.sh

@@ -35,6 +35,8 @@ readonly patches_repo_dir="${GOPATH}/src/${patches_repo}"
 readonly default_patches_dir="${patches_repo_dir}/kernel/patches/"
 # Default path to search config for kata
 readonly default_kernel_config_dir="${GOPATH}/src/${kernel_config_repo}/kernel/configs"
+# Default path to search for kernel config fragments
+readonly default_config_frags_dir="${GOPATH}/src/${kernel_config_repo}/kernel/configs/fragments"
 #Path to kernel directory
 kernel_path=""
 #
@@ -136,19 +138,95 @@ get_major_kernel_version() {
 	echo "${major_version}.${minor_version}"
 }
 
+# Make a kernel config file from generic and arch specific
+# fragments
+# - arg1 - path to arch specific fragments
+# - arg2 - path to kernel sources
+#
+get_kernel_frag_path() {
+	local arch_path="$1"
+	local common_path="${arch_path}/../common"
+	local kernel_path="$2"
+	local cmdpath="${kernel_path}/scripts/kconfig/merge_config.sh"
+	local config_path="${arch_path}/.config"
+
+	local arch_configs="$(ls ${arch_path}/*.conf)"
+	local common_configs="$(ls ${common_path}/*.conf)"
+
+	# These are the strings that the kernel merge_config.sh script kicks out
+	# when it reports an error or warning condition. We search for them in the
+	# output to try and fail when we think something has been misconfigured.
+	local not_in_string="not in final"
+	local redefined_string="not in final"
+	local redundant_string="not in final"
+
+	# Later, if we need to add kernel version specific subdirs in order to
+	# handle specific cases, then add the path definition and search/list/cat
+	# here.
+	local all_configs="${common_configs} ${arch_configs}"
+
+	info "Constructing config from fragments: ${config_path}"
+	local results=$(export KCONFIG_CONFIG=${config_path}; \
+					export ARCH=${arch_target}; \
+					cd ${kernel_path}; ${cmdpath} -r -n ${all_configs})
+
+	# Did we request any entries that did not make it?
+	local missing=$(echo $results | grep -v -q "${not_in_string}"; echo $?)
+	if [ ${missing} -ne 0 ]; then
+		info "Some CONFIG elements failed to make the final .config:"
+		info "${results}"
+		info "Generated config file can be found in ${config_path}"
+		die "Failed to construct requested .config file"
+	fi
+
+	# Did we define something as two different values?
+	local redefined=$(echo ${results} | grep -v -q "${redefined_string}"; echo $?)
+	if [ ${redefined} -ne 0 ]; then
+		info "Some CONFIG elements are redefined in fragments:"
+		info "${results}"
+		info "Generated config file can be found in ${config_path}"
+		die "Failed to construct requested .config file"
+	fi
+
+	# Did we define something twice? Nominally this may not be an error, and it
+	# might be convenient to allow it, but for now, let's pick up on them.
+	local redundant=$(echo ${results} | grep -v -q "${redundant_string}"; echo $?)
+	if [ ${redundant} -ne 0 ]; then
+		info "Some CONFIG elements failed to make the final .config"
+		info "${results}"
+		info "Generated config file can be found in ${config_path}"
+		die "Failed to construct requested .config file"
+	fi
+
+	echo "${config_path}"
+}
+
+# Locate and return the path to the relevant kernel config file
+# - arg1: kernel version
+# - arg2: hypervisor target
+# - arg3: arch target
+# - arg4: kernel source path
 get_default_kernel_config() {
 	local version="${1}"
 
 	local hypervisor="$2"
 	local kernel_arch="$3"
+	local kernel_path="$4"
 
 	[ -n "${version}" ] || die "kernel version not provided"
 	[ -n "${hypervisor}" ] || die "hypervisor not provided"
 	[ -n "${kernel_arch}" ] || die "kernel arch not provided"
 
 	local kernel_ver
 	kernel_ver=$(get_major_kernel_version "${version}")
-	config="${default_kernel_config_dir}/${kernel_arch}_kata_${hypervisor}_${major_kernel}.x"
+
+	archfragdir="${default_config_frags_dir}/${kernel_arch}"
+	if [ -d "${archfragdir}" ]; then
+		config="$(get_kernel_frag_path ${archfragdir} ${kernel_path})"
+	else
+		config="${default_kernel_config_dir}/${kernel_arch}_kata_${hypervisor}_${major_kernel}.x"
+	fi
+
 	[ -f "${config}" ] || die "failed to find default config ${config}"
 	echo "${config}"
 }
@@ -214,8 +292,9 @@ setup_kernel() {
 		done
 
 		[ -n "${hypervisor_target}" ] || hypervisor_target="kvm"
-		[ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}")
+		[ -n "${kernel_config_path}" ] || kernel_config_path=$(get_default_kernel_config "${kernel_version}" "${hypervisor_target}" "${arch_target}" "${kernel_path}")
 
+		info "Copying config file from: ${kernel_config_path}"
 		cp "${kernel_config_path}" ./.config
 		make oldconfig
 	)

kernel/configs/README.md

@@ -1,21 +1,65 @@
-## How to use config files
+* [Kata Containers kernel config files](#kata-containers-kernel-config-files)
+   * [Types of config files](#types-of-config-files)
+   * [How to use config files](#how-to-use-config-files)
+   * [How to modify config files](#how-to-modify-config-files)
 
-config files must be copied in the kernel source code directory and renamed to `.config`
+# Kata Containers kernel config files
 
-For example:
+This directory contains Linux Kernel config files used to configure Kata
+Containers VM kernels.
 
+## Types of config files
+
+This directory holds config files for the Kata Linux Kernel in two forms:
+
+- A tree of config file 'fragments' in the `fragments` sub-folder, that are
+  constructed into a complete config file using the kernel
+  `scripts/kconfig/merge_config.sh` script.
+- As complete config files that can be used as-is.
+
+Kernel config fragments are the preferred method of constructing `.config` files
+to build Kata Containers kernels, due to their improved clarity and ease of maintenance
+over single file monolithic `.config`s.
+
+## How to use config files
+
+The recommended way to set up a kernel tree, populate it with a relevant `.config` file,
+and build a kernel, is to use the [`build_kernel.sh`](../build-kernel.sh) script. For
+example:
+
+```bash
+$ ./build-kernel.sh setup
 ```
-cp x86_kata_kvm_4.14.x linux-4.14.22/.config
-pushd linux-4.14.22
-make ARCH=x86_64 -j4
-```
+
+The `build-kernel.sh` script understands both full and fragment based config files.
+
+Run `./build-kernel.sh help` for more information.
 
 ## How to modify config files
 
+Complete config files can be modified either with an editor, or preferably
+using the kernel `Kconfig` configuration tools, for example:
+
 ```
-cp x86_kata_kvm_4.14.x linux-4.14.22/.config
-pushd linux-4.14.22
-make menuconfig
-popd
-cp linux-4.14.22/.config x86_kata_kvm_4.14.x
+$ cp x86_kata_kvm_4.14.x linux-4.14.22/.config
+$ pushd linux-4.14.22
+$ make menuconfig
+$ popd
+$ cp linux-4.14.22/.config x86_kata_kvm_4.14.x
 ```
+
+Kernel fragments are best constructed using an editor. Tools such as `grep` and
+`diff` can help find the differences between two config files to be placed
+into a fragment.
+
+If adding config entries for a new subsystem or feature, consider making a new
+fragment with an appropriately descriptive name.
+
+The fragment gathering tool perfoms some basic sanity checks, and the `build-kernel.sh` will
+fail and report the error in the cases of:
+
+- A duplicate `CONFIG` symbol appearing.
+- A `CONFIG` symbol being in a fragment, but not appearing in the final .config
+  - which indicates that `CONFIG` variable is not a part of the kernel `Kconfig` setup, which
+    can indicate a typing mistake in the name of the symbol.
+- A `CONFIG` symbol appearing in the fragments with multiple different values.

kernel/configs/fragments/common/9p.conf

@@ -0,0 +1,17 @@
+# Enable 9p(fs) support - required for Kata to mount filesystems into the workload
+
+CONFIG_NET_9P=y
+CONFIG_NET_9P_VIRTIO=y
+CONFIG_9P_FS=y
+# NOTE - 9p client cacheing turned off?
+# FIXME: check if that is right?
+# https://github.com/kata-containers/packaging/issues/483
+#CONFIG_9P_FSCACHE=y
+CONFIG_NETWORK_FILESYSTEMS=y
+# Q. Do we use the POSIX_ACL over 9p?
+# FIXME: https://github.com/kata-containers/packaging/issues/483
+CONFIG_9P_FS_POSIX_ACL=y
+# NOTE - this adds security labels, such as used by SELinux - we may be able to
+# disable this, for now.
+# FIXME: https://github.com/kata-containers/packaging/issues/483
+CONFIG_9P_FS_SECURITY=y

kernel/configs/fragments/common/acpi.conf

@@ -0,0 +1,28 @@
+# enable ACPI support.
+# This could do with REVIEW
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_ARCH_SUPPORTS_ACPI=y
+CONFIG_ACPI=y
+CONFIG_ACPI_LEGACY_TABLES_LOOKUP=y
+CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC=y
+CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT=y
+CONFIG_ACPI_LPIT=y
+CONFIG_ACPI_BUTTON=y
+CONFIG_ACPI_CPU_FREQ_PSS=y
+CONFIG_ACPI_PROCESSOR_CSTATE=y
+CONFIG_ACPI_PROCESSOR_IDLE=y
+# Having trouble enabling this - disable for now.
+# Would add support for ACPI CPPC power control via firmware - do we need
+# that for the guest??
+#CONFIG_ACPI_CPPC_LIB=y
+CONFIG_ACPI_PROCESSOR=y
+CONFIG_ACPI_HOTPLUG_CPU=y
+CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_PCI_SLOT=y
+CONFIG_ACPI_CONTAINER=y
+CONFIG_ACPI_HOTPLUG_MEMORY=y
+CONFIG_ACPI_HOTPLUG_IOAPIC=y
+CONFIG_ACPI_NFIT=y
+CONFIG_HAVE_ACPI_APEI=y
+CONFIG_HAVE_ACPI_APEI_NMI=y

kernel/configs/fragments/common/base.conf

@@ -0,0 +1,57 @@
+# Basic necessary items!
+
+CONFIG_SMP=y
+CONFIG_HYPERVISOR_GUEST=y
+CONFIG_PARAVIRT=y
+CONFIG_KVM_GUEST=y
+# Note, no nested VM support enabled here
+
+# Turn off embedded mode, as it disabled 'too much', and we
+# no longer pass all the tests. We should refine this, and
+# work out which of the ~66 items it enables are really needed.
+# I believe this is the actual syntax we need for a fragment to
+# disable an item...
+# CONFIG_EMBEDDED is not set
+
+# Note, no virt enabled baloon yet
+CONFIG_INPUT=y
+CONFIG_PRINTK=y
+# We use this for metrics!
+CONFIG_PRINTK_TIME=y
+CONFIG_UNIX98_PTYS=y
+CONFIG_FUTEX=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_GENERIC_MSI_IRQ_DOMAIN=y
+CONFIG_GENERIC_MSI_IRQ=y
+CONFIG_LEGACY_VSYSCALL_NONE=y
+CONFIG_NO_HZ=y
+CONFIG_NO_HZ_FULL=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_POSIX_TIMERS=y
+CONFIG_PROC_SYSCTL=y
+
+CONFIG_SHMEM=y
+
+# For security...
+CONFIG_RELOCATABLE=y
+# FIXME - check if we should be setting this
+# https://github.com/kata-containers/packaging/issues/483
+#CONFIG_RANDOMIZE_BASE=y
+# FIXME - check if we should be setting this
+# https://github.com/kata-containers/packaging/issues/483
+# I have a feeling it effects our memory hotplug maybe?
+# PHYSICAL_ALIGN=0x1000000
+CONFIG_RETPOLINE=y
+
+# This would only affect two drivers, neither of which we have enabled.
+# The recommendation is to have it on, and you will see if in a diff if you
+# look for differences against the frag generated config - so, add it here as
+# a comment to make it clear in the future why we have not set it - as it would
+# only add noise to our frags and config.
+# PREVENT_FIRMWARE_BUILD=y
+
+# Trust the hardware vendor to initialise the RNG - which can speed up boot.
+# This can still be dynamically disabled on the kernel command line/kata config if needed.
+# Disable for now, as it upsets the entropy test, and we need to improve those: FIXME: see:
+# https://github.com/kata-containers/tests/issues/1543
+# CONFIG_RANDOM_TRUST_CPU is not set

kernel/configs/fragments/common/cgroup.conf

@@ -0,0 +1,22 @@
+# Add cgroup support. Needed both for the agent to place the workload into, and
+# also used/looked for by systemd rootfs.
+CONFIG_CGROUPS=y
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CGROUP_WRITEBACK=y
+CONFIG_CGROUP_SCHED=y
+CONFIG_FAIR_GROUP_SCHED=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_SOCK_CGROUP_DATA=y
+
+# We have to enable SWAP CG, as runc/libcontainer in the agent currently fails
+# to write to it, even though it does some checks to see if swap is enabled.
+CONFIG_SWAP=y
+CONFIG_MEMCG_SWAP=y
+CONFIG_MEMCG_SWAP_ENABLED=y

kernel/configs/fragments/common/cpu.conf

@@ -0,0 +1,7 @@
+# Items to do with CPU frequency, power etc.
+
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_MENU=y

kernel/configs/fragments/common/crypto.conf

@@ -0,0 +1,15 @@
+# Need decompressors for root filesystems and kernels.
+# Do we need all of these?
+CONFIG_CRYPTO=y
+# Deflate used by IPSec and IPCOMP protocols
+# Also selects ZLIB and a couple of other algos
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_XZ_DEC=y
+CONFIG_ZLIB_DEFLATE=y
+# FIXME - check, do we need gzip?
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_DECOMPRESS_GZIP=y
+# Some items required by systemd: https://github.com/systemd/systemd/blob/master/README
+CONFIG_CRYPTO_USER_API=y
+CONFIG_CRYPTO_USER_API_HASH=y
+CONFIG_CRYPTO_SHA256=y

kernel/configs/fragments/common/dax.conf

@@ -0,0 +1,37 @@
+# Enable DAX and NVDIMM support so we can map in our rootfs
+
+# Need HOTREMOVE, or ZONE_DEVICE will not get enabled
+# We don't actually afaik remove any memory once we have plugged it in, as
+# generally it is too 'expensive' an operation.
+CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
+CONFIG_MEMORY_HOTREMOVE=y
+# Also need this
+CONFIG_SPARSEMEM_VMEMMAP=y
+# And this should be auto set by the arch already
+CONFIG_ARCH_HAS_ZONE_DEVICE=y
+
+# Without these the pmem_should_map_pages() call in the kernel fails with new
+# Related to the ARCH_HAS_HMM set in the arch files.
+CONFIG_ZONE_DEVICE=y
+CONFIG_DEV_PAGEMAP_OPS=y
+
+CONFIG_ND_PFN=y
+CONFIG_NVDIMM_PFN=y
+CONFIG_NVDIMM_DAX=y
+
+CONFIG_RADIX_TREE_MULTIORDER=y
+
+CONFIG_BLOCK=y
+CONFIG_BLK_DEV=y
+CONFIG_BLK_DEV_PMEM=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_LIBNVDIMM=y
+CONFIG_ND_BLK=y
+CONFIG_BTT=y
+# FIXME: Should check if this is really needed
+# https://github.com/kata-containers/packaging/issues/483
+CONFIG_NVMEM=y
+# Is auto selected by other options
+#CONFIG_DAX_DRIVER=y
+CONFIG_DAX=y
+CONFIG_FS_DAX=y

kernel/configs/fragments/common/elf.conf

@@ -0,0 +1,5 @@
+# Enable Elf loading, and script loading
+
+CONFIG_BINFMT_ELF=y
+CONFIG_BINFMT_SCRIPT=y
+CONFIG_BINFMT_MISC=y