WIP - Enable SELinux

Vagrantfile

@@ -1,32 +1,34 @@
-BOX = "generic/alpine310"
+OS = (ENV['OS'] || "centos7" )
+BOX = (ENV['box'] || "generic/#{OS}")
 HOME = File.dirname(__FILE__)
 PROJECT = File.basename(HOME)
-MOUNT_TYPE = ENV['MOUNT_TYPE'] || "nfs"
 NUM_NODES = (ENV['NUM_NODES'] || 0).to_i
 NODE_CPUS = (ENV['NODE_CPUS'] || 4).to_i
 NODE_MEMORY = (ENV['NODE_MEMORY'] || 8192).to_i
 NETWORK_PREFIX = ENV['NETWORK_PREFIX'] || "10.135.135"
-VAGRANT_PROVISION = ENV['VAGRANT_PROVISION'] || "./scripts/vagrant-provision"
+VAGRANT_PROVISION = ENV['VAGRANT_PROVISION'] || "./scripts/#{BOX}/vagrant-provision"
+MOUNT_TYPE = ENV['MOUNT_TYPE'] || ""
+# MOUNT_TYPE = ENV['MOUNT_TYPE'] || "nfs"
 
 # --- Rules for /etc/sudoers to avoid password entry configuring NFS:
 # %admin	ALL = (root) NOPASSWD: /usr/bin/sed -E -e * -ibak /etc/exports
 # %admin	ALL = (root) NOPASSWD: /usr/bin/tee -a /etc/exports
 # %admin	ALL = (root) NOPASSWD: /sbin/nfsd restart
 # --- May need to add terminal to System Preferences -> Security & Privacy -> Privacy -> Full Disk Access
 
-# --- Check for missing plugins
-required_plugins = %w( vagrant-alpine vagrant-timezone )
-plugin_installed = false
-required_plugins.each do |plugin|
-  unless Vagrant.has_plugin?(plugin)
-    system "vagrant plugin install #{plugin}"
-    plugin_installed = true
-  end
-end
-# --- If new plugins installed, restart Vagrant process
-if plugin_installed === true
-  exec "vagrant #{ARGV.join' '}"
-end
+# # --- Check for missing plugins
+# required_plugins = %w( vagrant-alpine vagrant-timezone )
+# plugin_installed = false
+# required_plugins.each do |plugin|
+#  unless Vagrant.has_plugin?(plugin)
+#    system "vagrant plugin install #{plugin}"
+#    plugin_installed = true
+#  end
+# end
+# # --- If new plugins installed, restart Vagrant process
+# if plugin_installed === true
+#  exec "vagrant #{ARGV.join' '}"
+# end
 
 provision = <<SCRIPT
 # --- Use system gopath if available
@@ -61,8 +63,9 @@ Vagrant.configure("2") do |config|
   config.vm.hostname = PROJECT
   config.vm.synced_folder ".", HOME, type: MOUNT_TYPE
   config.vm.provision "shell", inline: provision
-  config.timezone.value = :host
-
+  if Vagrant.has_plugin?("vagrant-timezone")
+    config.timezone.value = :host
+  end
   config.vm.network "private_network", ip: "#{NETWORK_PREFIX}.100" if NUM_NODES==0
 
   (1..NUM_NODES).each do |i|

go.mod

@@ -9,7 +9,7 @@ replace (
 	github.com/containerd/console => github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
 	github.com/containerd/containerd => github.com/rancher/containerd v1.3.3-k3s1
 	github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02
-	github.com/containerd/cri => github.com/rancher/cri v1.3.0-k3s.3
+	github.com/containerd/cri => github.com/erikwilson/rancher-cri v1.11.1-0.20200221204331-01cec633ed04
 	github.com/containerd/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
 	github.com/containerd/go-runc => github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda
 	github.com/containerd/typeurl => github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
@@ -124,6 +124,5 @@ require (
 	k8s.io/component-base v0.0.0
 	k8s.io/cri-api v0.0.0
 	k8s.io/klog v1.0.0
-	k8s.io/kubelet v0.0.0
 	k8s.io/kubernetes v1.16.0
 )

go.sum

@@ -229,6 +229,8 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikwilson/rancher-cri v1.11.1-0.20200221204331-01cec633ed04 h1:t4+xCLqiYyA8BGUDnu54QzaLwO94Iqa2VtSwSPIQjmk=
+github.com/erikwilson/rancher-cri v1.11.1-0.20200221204331-01cec633ed04/go.mod h1:GfpXQiZC0mC9ik1rXKuispIvYhBGnXun3El0abjeUYI=
 github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY=
 github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -707,8 +709,6 @@ github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 h1:83l9gPhYtgxODl
 github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8/go.mod h1:4XHkfaUj+URzGO9sohoAgt2V9Y8nIW7fugpu0E6gShk=
 github.com/rancher/containerd v1.3.3-k3s1 h1:j8NGZdXKsZd2ne0XQg6OBfMJ/NkY/Qri6QhscGrJp2M=
 github.com/rancher/containerd v1.3.3-k3s1/go.mod h1:ZMfzmqce2Z+QSEqdHMfeJs1TZ/UeJ1aDrazjpQT4ehM=
-github.com/rancher/cri v1.3.0-k3s.3 h1:j/Sq2LMyg6gBn2MS1j5dEudpdL+UYVH7ubbewUCXkS0=
-github.com/rancher/cri v1.3.0-k3s.3/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY=
 github.com/rancher/cri-tools v1.17.0-k3s1 h1:jfu97FowbraTDc7b6fxWtO+dq+DU2oW+ABBQSEFiRb0=
 github.com/rancher/cri-tools v1.17.0-k3s1/go.mod h1:bRTZttsvk+nCG8tSFs8D6UUx8CkMXR5TAsRLS0fXAqI=
 github.com/rancher/dynamiclistener v0.2.0 h1:KucYwJXVVGhZ/NndfMCeQoCafT/VN7kvqSGgmlX8Lxk=

install.sh

@@ -396,15 +396,15 @@ setup_binary() {
     $SUDO chown root:root ${TMP_BIN}
     $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s
 
-    if command -v getenforce > /dev/null 2>&1; then
-        if [ "Disabled" != $(getenforce) ]; then
-	    info 'SELinux is enabled, setting permissions'
-	    if ! $SUDO semanage fcontext -l | grep "${BIN_DIR}/k3s" > /dev/null 2>&1; then
-	        $SUDO semanage fcontext -a -t bin_t "${BIN_DIR}/k3s"
-	    fi
-	    $SUDO restorecon -v ${BIN_DIR}/k3s > /dev/null
-        fi
-    fi
+    # if command -v getenforce > /dev/null 2>&1; then
+    #     if [ "Disabled" != $(getenforce) ]; then
+	#     info 'SELinux is enabled, setting permissions'
+	#     if ! $SUDO semanage fcontext -l | grep "${BIN_DIR}/k3s" > /dev/null 2>&1; then
+	#         $SUDO semanage fcontext -a -t bin_t "${BIN_DIR}/k3s"
+	#     fi
+	#     $SUDO restorecon -v ${BIN_DIR}/k3s > /dev/null
+    #     fi
+    # fi
 }
 
 # --- download and verify k3s ---
@@ -520,6 +520,7 @@ do_unmount() {
 }
 
 do_unmount '/run/k3s'
+do_unmount '/run/containerd'
 do_unmount '/var/lib/rancher/k3s'
 do_unmount '/var/lib/kubelet/pods'
 do_unmount '/run/netns/cni-'

pkg/agent/config/config.go

@@ -423,15 +423,17 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
 	nodeConfig.AgentConfig.StrongSwanDir = filepath.Join(envInfo.DataDir, "strongswan")
 	nodeConfig.CACerts = info.CACerts
-	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
-	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
-	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "containerd")
-	if !envInfo.Debug {
-		nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "containerd/containerd.log")
-	}
-	nodeConfig.Containerd.State = "/run/k3s/containerd"
-	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
-	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
+
+	nodeConfig.Containerd.Config = "/etc/containerd/config.toml"
+	nodeConfig.Containerd.Root = "/var/lib/containerd"
+	nodeConfig.Containerd.Opt = "/opt/containerd"
+	nodeConfig.Containerd.Log = "/var/log/containerd.log"
+	// nodeConfig.Containerd.State = "/var/lib/docker/overlay2/"
+	nodeConfig.Containerd.State = "/run/containerd"
+	nodeConfig.Containerd.Address = "/run/containerd/containerd.sock"
+	nodeConfig.Containerd.Log = "/var/log/containerd.log"
+	nodeConfig.Containerd.Template = "/etc/containerd/config.toml.tmpl"
+
 	nodeConfig.ServerAddress = serverURLParsed.Host
 	nodeConfig.Certificate = servingCert
 

pkg/agent/templates/templates.go

@@ -20,6 +20,7 @@ const ContainerdConfigTemplate = `
 [plugins.cri]
   stream_server_address = "127.0.0.1"
   stream_server_port = "10010"
+  enable_selinux = true
 
 {{- if .IsRunningInUserNS }}
   disable_cgroup = true

pkg/ctr/main.go

@@ -35,7 +35,7 @@ func main() {
 	for i, flag := range app.Flags {
 		if sFlag, ok := flag.(cli.StringFlag); ok {
 			if sFlag.Name == "address, a" {
-				sFlag.Value = "/run/k3s/containerd/containerd.sock"
+				sFlag.Value = "/run/containerd/containerd.sock"
 				app.Flags[i] = sFlag
 			} else if sFlag.Name == "namespace, n" {
 				sFlag.Value = "k8s.io"

scripts/build

@@ -36,6 +36,7 @@ LDFLAGS="
 STATIC="
     -extldflags '-static'
 "
+RUNC_TARGET="static"
 
 if [ "$DQLITE" = "true" ]; then
     DQLITE_TAGS="dqlite"
@@ -45,13 +46,14 @@ fi
 STATIC_SQLITE="
     -extldflags '-static -lm -ldl -lz -lpthread $DQLITE_STATIC_SQLITE'
 "
-TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless $DQLITE_TAGS"
+TAGS="ctrd apparmor seccomp selinux no_btrfs netcgo osusergo providerless $DQLITE_TAGS"
 
 if [ "$STATIC_BUILD" != "true" ]; then
     STATIC="
 "
     STATIC_SQLITE="
 "
+    RUNC_TARGET=""
 else
     TAGS="static_build libsqlite3 $TAGS"
 fi
@@ -109,13 +111,16 @@ ln -s containerd ./bin/ctr
 # echo Building containerd
 # CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
 echo Building runc
-make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static
+rm -f ./vendor/github.com/opencontainers/runc/runc
+make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp selinux" -C ./vendor/github.com/opencontainers/runc $RUNC_TARGET
 cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc
 
 echo Building containerd-shim
+rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim
 make -C ./vendor/github.com/containerd/containerd bin/containerd-shim
 cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim ./bin/containerd-shim
 
 echo Building containerd-shim-runc-v2
+rm -f ./vendor/github.com/containerd/containerd/bin/containerd-shim-runc-v2
 make -C ./vendor/github.com/containerd/containerd bin/containerd-shim-runc-v2
 cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim-runc-v2 ./bin/containerd-shim-runc-v2

scripts/vagrant-provision → scripts/generic/alpine310/vagrant-provision

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ve
 
-cd $(dirname $0)/..
+cd $(dirname $0)/../../..
 
 if [ ! -f /etc/vagrant_box_build_time ]; then
     echo "This script should only be called during vagrant provisioning"

scripts/generic/centos7/vagrant-provision

@@ -0,0 +1,95 @@
+#!/bin/bash
+set -ve
+
+cd $(dirname $0)/../../..
+
+if [ ! -f /etc/vagrant_box_build_time ]; then
+    echo "This script should only be called during vagrant provisioning"
+    exit 1
+fi
+
+ARCH=amd64
+if [[ $HOME == /go/* ]]; then
+    echo "Must not launch vagrant from /go/"
+    exit 1
+fi 
+
+if [ -z "${GOPATH}" ]; then
+    GOPATH=$(realpath $HOME/../../../..)
+    echo "WARNING: Assuming GOPATH=$GOPATH"
+else
+    echo "Using parent GOPATH=$GOPATH"
+fi
+goversion=$(grep "^FROM " Dockerfile.dapper | sed -e 's/^FROM golang:\(.*\)-.*/\1/')
+
+if [ -z "$goversion" ]; then
+    echo "Cannot find version of go to fetch"
+    exit 1
+fi
+
+echo "Installing go $goversion"
+curl -sL https://storage.googleapis.com/golang/go${goversion}.linux-${ARCH}.tar.gz | tar -xzf - -C /usr/local
+# ---
+cat <<EOF >/etc/profile.d/build.sh
+export GO111MODULE=off
+export GOPATH=$GOPATH
+export PATH=/usr/local/bin:$PATH:/usr/local/go/bin:$GOPATH/bin
+EOF
+. /etc/profile.d/build.sh
+# ---
+sed -i 's|:/bin/ash$|:/bin/bash|g' /etc/passwd
+cat <<\EOF >/etc/profile.d/color.sh
+alias ls='ls --color=auto'
+export PS1='\033[31m[ \033[90m\D{%F %T}\033[31m ]\n\[\033[36m\]\u\[\033[m\]🐮\[\033[32m\]\h:\[\033[33;1m\]\w\[\033[m\]$ '
+EOF
+# ---
+yum install -y \
+    bash git gcc musl-dev vim less file wget jq tar zip squashfs-tools npm coreutils \
+    libffi-dev libseccomp-devel make \
+    pkg-config libselinux-devel \
+    container-selinux policycoreutils policycoreutils-python
+# ---
+yum install -y gdb
+go get -u github.com/go-delve/delve/cmd/dlv
+# ---
+# docker install instructions slightly changed from https://kubernetes.io/docs/setup/production-environment/container-runtimes/
+# default "exec-opts": ["native.cgroupdriver=cgroupfs"], and set "selinux-enabled": true
+yum install -y yum-utils device-mapper-persistent-data lvm2
+yum-config-manager --add-repo   https://download.docker.com/linux/centos/docker-ce.repo
+yum update -y && yum install -y   containerd.io-1.2.10   docker-ce-19.03.4   docker-ce-cli-19.03.4
+mkdir -p /etc/docker
+cat > /etc/docker/daemon.json <<EOF
+{
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "selinux-enabled": true,
+  "storage-driver": "overlay2",
+  "storage-opts": [
+    "overlay2.override_kernel_check=true"
+  ]
+}
+EOF
+# ---
+systemctl daemon-reload
+systemctl disable --now firewalld
+# disable docker for now as the default containerd path interferes
+systemctl disable --now docker
+# ---
+# set selinux to permissive for dev & testing purposes only
+setenforce 0
+sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
+# ---
+cat <<\EOF >>/etc/motd
+               ,        ,
+   ,-----------|'------'| |\    ____
+  /.           '-'@  o|-' | |  /___ \
+ |/|             | .. |   | | __ __) | ____
+   |   .________.'----'   | |/ /|__ < / __/
+   |  ||        |  ||     |   < ___) |\__ \
+   \__|'        \__|'     |_|\_\_____/____/
+
+EOF
+# ---
+set +v