Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enforcer(bpf): automatically mount bpffs #1097

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion KubeArmor/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ type KubearmorConfig struct {

CoverageTest bool // Enable/Disable Coverage Test

LsmOrder []string // LSM order
LsmOrder []string // LSM order
BPFFsPath string // path to the BPF filesystem
}

// PolicyDir policy dir path for host policies backup
Expand Down Expand Up @@ -115,6 +116,9 @@ const ConfigK8sEnv string = "k8s"
// LsmOrder Preference order of the LSMs
const LsmOrder string = "lsm"

// BPFFsPath key
const BPFFsPath string = "bpfFsPath"

func readCmdLineParams() {
hostname, _ := os.Hostname()
clusterStr := flag.String(ConfigCluster, "default", "cluster name")
Expand Down Expand Up @@ -145,6 +149,8 @@ func readCmdLineParams() {

lsmOrder := flag.String(LsmOrder, "bpf,apparmor,selinux", "lsm preference order to use, available lsms [bpf, apparmor, selinux]")

bpfFsPath := flag.String(BPFFsPath, "/sys/fs/bpf", "Path to the BPF filesystem to use for storing maps")

flags := []string{}
flag.VisitAll(func(f *flag.Flag) {
kv := fmt.Sprintf("%s:%v", f.Name, f.Value)
Expand Down Expand Up @@ -181,6 +187,8 @@ func readCmdLineParams() {
viper.SetDefault(ConfigCoverageTest, *coverageTestB)

viper.SetDefault(LsmOrder, *lsmOrder)

viper.SetDefault(BPFFsPath, *bpfFsPath)
}

// LoadConfig Load configuration
Expand Down Expand Up @@ -257,6 +265,8 @@ func LoadConfig() error {

GlobalCfg.LsmOrder = strings.Split(viper.GetString(LsmOrder), ",")

GlobalCfg.BPFFsPath = viper.GetString(BPFFsPath)

kg.Printf("Final Configuration [%+v]", GlobalCfg)

return nil
Expand Down
228 changes: 228 additions & 0 deletions KubeArmor/enforcer/bpflsm/bpffs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2022 Authors of KubeArmor

package bpflsm
daemon1024 marked this conversation as resolved.
Show resolved Hide resolved

// This implementation for automounting bpffs has been inspired from
// Cilium - https://github.com/cilium/cilium/blob/master/pkg/bpf/bpffs_linux.go

import (
"fmt"
"os"
"sync"

"github.com/cilium/cilium/pkg/mountinfo"
"golang.org/x/sys/unix"
)

var (
// the default BPFFs mountpoint
defaultBPFFsPath = "/sys/fs/bpf"

// directory to fallback upon if default bpffs directory on host
// had some other filesystem mounted
fallbackBPFFsPath = "/run/kubearmor/bpffs"

// path to where BPFFs is mounted
// this is the final location where enforcer will save maps
// after checking different mount points
mapRoot = defaultBPFFsPath

// for detecting misorder
lockedDown = false
once sync.Once
mountOnce sync.Once
)

// CheckOrMountBPFFs checks for the mounted BPF filesystem at either
// the standard or the user specified custom location.
// - No custom location specified, check if BPFFS mounted at
// /sys/fs/bpf
// - No - Mount BPFFS at /sys/fs/bpf
// - Yes - We're done
// - Yes but /sys/fs/bpf has a different fs mounted which implies
// that kubearmor is running inside a container and the host
// mount is an empty directory. So we mount BPFFS under
// /run/kubearmor/bpffs.
// - Custom location specified, check if BPFFS is mounted there
// - No - Mount it
// - Yes - We're done
// - Yes but the location has some different fs mounted, return
// an error
//
// We also check and error if there have been multiple mounts at
// the same point. See - https://patchwork.kernel.org/project/netdevbpf/patch/20220223131833.51991-1-laoar.shao@gmail.com/
func (be *BPFEnforcer) CheckOrMountBPFFs(bpfRoot string) {
mountOnce.Do(func() {
if err := be.checkOrMountBPFFs(bpfRoot); err != nil {
be.Logger.Err("Unable to mount BPF filesystem")
}
})
}

func (be *BPFEnforcer) checkOrMountBPFFs(bpfRoot string) error {
if bpfRoot == defaultBPFFsPath {
// mount BPFFs at the default path
if err := be.checkOrMountDefaultLocations(); err != nil {
return err
}
} else {
// the user specified a custom path for BPFFs
if err := be.checkOrMountCustomLocation(bpfRoot); err != nil {
return err
}
}

multipleMounts, err := hasMultipleMounts()
if err != nil {
return err
}
if multipleMounts {
return fmt.Errorf("multiple mount points detected at %s", mapRoot)
}

return nil
}

func (be *BPFEnforcer) checkOrMountDefaultLocations() error {
// Check whether /sys/fs/bpf has a BPFFS mount.
mounted, bpffsInstance, err := mountinfo.IsMountFS(mountinfo.FilesystemTypeBPFFS, mapRoot)
if err != nil {
return err
}

// If /sys/fs/bpf is not mounted at all, we should mount
// BPFFS there.
if !mounted {
be.Logger.Printf("Mounting BPF Filesystem at %s", mapRoot)
if err := mountFs(); err != nil {
return err
}

return nil
}

if !bpffsInstance {
// If /sys/fs/bpf has a mount but with some other filesystem
// than BPFFS, it means that Kubearmor is running inside
// container and /sys/fs/bpf is not mounted on host. So, we
// mount BPFFS in /run/kubearmor/bpffs inside the container.
// This will allow operation of Kubearmor but will result in
// unmounting of the filesystem when the pod is restarted.
be.Logger.Warnf("BPF filesystem is going to be mounted automatically "+
"in %s. However, it probably means that Kubearmor is running "+
"inside container and BPFFS is not mounted on the host. ",
mapRoot)

if lockedDown {
setMapRoot(mapRoot)
}

cMounted, cBpffsInstance, err := mountinfo.IsMountFS(mountinfo.FilesystemTypeBPFFS, mapRoot)
if err != nil {
return err
}
if !cMounted {
be.Logger.Printf("Mounting BPF Filesystem at %s", mapRoot)
if err := mountFs(); err != nil {
return err
}
} else if !cBpffsInstance {
be.Logger.Printf("%s is mounted but has a different filesystem than BPFFS", fallbackBPFFsPath)
}
}

be.Logger.Printf("Detected mounted BPF filesystem at %s", mapRoot)

return nil
}

func (be *BPFEnforcer) checkOrMountCustomLocation(bpfRoot string) error {
setMapRoot(bpfRoot)

// Check whether the custom location has a BPFFS mount.
mounted, bpffsInstance, err := mountinfo.IsMountFS(mountinfo.FilesystemTypeBPFFS, bpfRoot)
if err != nil {
return err
}

// If the custom location has no mount, let's mount BPFFS there.
if !mounted {
setMapRoot(bpfRoot)
be.Logger.Printf("Mounting BPF Filesystem at %s", mapRoot)
if err := mountFs(); err != nil {
return err
}

return nil
}

// If the custom location already has a mount with some other filesystem than
// BPFFS, return the error.
if !bpffsInstance {
return fmt.Errorf("mount in the custom directory %s has a different filesystem than BPFFS", bpfRoot)
}

be.Logger.Printf("Detected mounted BPF filesystem at %s", mapRoot)

return nil
}

// hasMultipleMounts checks whether the current mapRoot has only one mount.
func hasMultipleMounts() (bool, error) {
num := 0

mountInfos, err := mountinfo.GetMountInfo()
if err != nil {
return false, err
}

for _, mountInfo := range mountInfos {
if mountInfo.Root == "/" && mountInfo.MountPoint == mapRoot {
num++
}
}

return num > 1, nil
}

// mounts BPFFS into mapRoot directory
func mountFs() error {
mapRootStat, err := os.Stat(mapRoot)
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(mapRoot, 0750); err != nil {
return fmt.Errorf("unable to create bpf mount directory: %s", err)
}
} else {
return fmt.Errorf("failed to stat the mount path %s: %s", mapRoot, err)

}
} else if !mapRootStat.IsDir() {
return fmt.Errorf("%s is a file which is not a directory", mapRoot)
}

if err := unix.Mount(mapRoot, mapRoot, "bpf", 0, ""); err != nil {
return fmt.Errorf("failed to mount %s: %s", mapRoot, err)
}
return nil
}

func lockDown() {
lockedDown = true
}

func setMapRoot(path string) {
// we don't want to change the path on which maps are stored to
// be changed once we start writing maps
if lockedDown {
panic("setMapRoot() call after MapRoot was read")
}
mapRoot = path
}

// GetMapRoot function returns the current mapRoot path
func GetMapRoot() string {
daemon1024 marked this conversation as resolved.
Show resolved Hide resolved
once.Do(lockDown)
return mapRoot
}
7 changes: 5 additions & 2 deletions KubeArmor/enforcer/bpflsm/enforcer.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/cilium/ebpf/link"
"github.com/cilium/ebpf/rlimit"

"github.com/kubearmor/KubeArmor/KubeArmor/config"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
fd "github.com/kubearmor/KubeArmor/KubeArmor/feeder"
tp "github.com/kubearmor/KubeArmor/KubeArmor/types"
Expand Down Expand Up @@ -64,6 +65,8 @@ func NewBPFEnforcer(node tp.Node, logger *fd.Feeder) (*BPFEnforcer, error) {
MaxEntries: 256,
}

be.CheckOrMountBPFFs(config.GlobalCfg.BPFFsPath)

be.BPFContainerMap, err = ebpf.NewMapWithOptions(&ebpf.MapSpec{
Type: ebpf.HashOfMaps,
KeySize: 8,
Expand All @@ -73,7 +76,7 @@ func NewBPFEnforcer(node tp.Node, logger *fd.Feeder) (*BPFEnforcer, error) {
InnerMap: be.InnerMapSpec,
Name: "kubearmor_containers",
}, ebpf.MapOptions{
PinPath: "/sys/fs/bpf",
PinPath: GetMapRoot(),
})
if err != nil {
be.Logger.Errf("error creating kubearmor_containers map: %s", err)
Expand All @@ -82,7 +85,7 @@ func NewBPFEnforcer(node tp.Node, logger *fd.Feeder) (*BPFEnforcer, error) {

if err := loadEnforcerObjects(&be.obj, &ebpf.CollectionOptions{
Maps: ebpf.MapOptions{
PinPath: "/sys/fs/bpf",
PinPath: GetMapRoot(),
},
}); err != nil {
be.Logger.Errf("error loading BPF LSM objects: %v", err)
Expand Down
18 changes: 9 additions & 9 deletions KubeArmor/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,26 @@ replace (
github.com/kubearmor/KubeArmor/KubeArmor/monitor => ./monitor
github.com/kubearmor/KubeArmor/KubeArmor/policy => ./policy
github.com/kubearmor/KubeArmor/KubeArmor/types => ./types
github.com/kubearmor/KubeArmor/pkg/KubeArmorController => ../pkg/KubeArmorController
github.com/kubearmor/KubeArmor/deployments => ../deployments
github.com/kubearmor/KubeArmor/pkg/KubeArmorController => ../pkg/KubeArmorController
github.com/kubearmor/KubeArmor/protobuf => ../protobuf
)

require (
github.com/Masterminds/sprig/v3 v3.2.2
github.com/cilium/ebpf v0.9.1
github.com/cilium/cilium v1.12.6
github.com/cilium/ebpf v0.10.0
github.com/containerd/containerd v1.6.8
github.com/containerd/typeurl v1.0.2
github.com/docker/docker v20.10.17+incompatible
github.com/google/uuid v1.3.0
github.com/kubearmor/KubeArmor/pkg/KubeArmorController v0.0.0-20230102134750-3c9ddc923c3f
github.com/kubearmor/KubeArmor/deployments v0.0.0-00010101000000-000000000000
github.com/kubearmor/KubeArmor/pkg/KubeArmorController v0.0.0-20230102134750-3c9ddc923c3f
github.com/kubearmor/KubeArmor/protobuf v0.0.0-20220908103453-7b92e248beb9
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
github.com/spf13/viper v1.13.0
go.uber.org/zap v1.23.0
golang.org/x/sys v0.3.0
google.golang.org/grpc v1.49.0
k8s.io/api v0.25.0
k8s.io/apimachinery v0.25.0
Expand Down Expand Up @@ -80,15 +82,14 @@ require (
github.com/mitchellh/reflectwalk v1.0.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.13.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
Expand All @@ -101,11 +102,10 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
golang.org/x/net v0.0.0-20220907135653-1e95f45603a7 // indirect
golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
golang.org/x/sys v0.0.0-20220907062415-87db552b00fd // indirect
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/term v0.3.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
Loading