{elf_reader,linker}: Add support for weak kfuncs

This commit allows users to mark their kfunc defintions as `__weak`.
Weak kfuncs do not cause an error during loading if the kfunc in
question can't be found. Instead, a poison value is written which will
cause the verifier to bail out if the instruction is evaluated.

In addition, this commit relocates LDIMM64 instructions with kfunc
relocation entries. When relocated, the BTF id of the kfunc is written
to the instruction's immediate field.

Together these two changes allow users to write eBPF programs which can
gracefully handle the absence of a kfunc. To do so the `bpf_ksym_exists`
macro from `bpf_helpers.h` can be used to check if a kfunc is present.

```
void invalid_kfunc(void) __ksym __weak;

__section("tp_btf/task_newtask") int weak_kfunc_missing(void *ctx) {
	if (bpf_ksym_exists(invalid_kfunc)) {
		invalid_kfunc();
		return 0;
	}

	return 1;
}
```

In this example the kfunc is always invalid, yet the program can still
load since the branch calling the kfunc is unreachable.

So this effectively provides CO-RE capabilities for kfuncs. Especially
important since kfuncs are less stable than helpers, their availability
depending on kernel version, kernel compilation flags, or kernel modules
being loaded or not.

Signed-off-by: Dylan Reimerink <dylan.reimerink@isovalent.com>

elf_reader.go

@@ -26,7 +26,12 @@ type kconfigMeta struct {
 	Offset uint32
 }
 
-type kfuncMeta struct{}
+type kfuncMetaKey struct{}
+
+type kfuncMeta struct {
+	Binding elf.SymBind
+	Func    *btf.Func
+}
 
 // elfCode is a convenience to reduce the amount of arguments that have to
 // be passed around explicitly. You should treat its contents as immutable.
@@ -600,7 +605,7 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err
 	// function declarations, as well as extern kfunc declarations using __ksym
 	// and extern kconfig variables declared using __kconfig.
 	case undefSection:
-		if bind != elf.STB_GLOBAL {
+		if bind != elf.STB_GLOBAL && bind != elf.STB_WEAK {
 			return fmt.Errorf("asm relocation: %s: %w: %s", name, errUnsupportedBinding, bind)
 		}
 
@@ -610,13 +615,25 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err
 
 		kf := ec.kfuncs[name]
 		switch {
-		// If a Call instruction is found and the datasec has a btf.Func with a Name
-		// that matches the symbol name we mark the instruction as a call to a kfunc.
+		// If a Call / DWordLoad instruction is found and the datasec has a btf.Func with a Name
+		// that matches the symbol name we mark the instruction as a referencing a kfunc.
 		case kf != nil && ins.OpCode.JumpOp() == asm.Call:
-			ins.Metadata.Set(kfuncMeta{}, kf)
+			ins.Metadata.Set(kfuncMetaKey{}, &kfuncMeta{
+				Func:    kf,
+				Binding: bind,
+			})
+
 			ins.Src = asm.PseudoKfuncCall
 			ins.Constant = -1
 
+		case kf != nil && ins.OpCode.IsDWordLoad():
+			ins.Metadata.Set(kfuncMetaKey{}, &kfuncMeta{
+				Func:    kf,
+				Binding: bind,
+			})
+
+			ins.Constant = 0
+
 		// If no kconfig map is found, this must be a symbol reference from inline
 		// asm (see testdata/loader.c:asm_relocation()) or a call to a forward
 		// function declaration (see testdata/fwd_decl.c). Don't interfere, These
@@ -626,6 +643,10 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err
 		// require it to contain the symbol to disambiguate between inline asm
 		// relos and kconfigs.
 		case ec.kconfig != nil && ins.OpCode.IsDWordLoad():
+			if bind != elf.STB_GLOBAL {
+				return fmt.Errorf("asm relocation: %s: %w: %s", name, errUnsupportedBinding, bind)
+			}
+
 			for _, vsi := range ec.kconfig.Value.(*btf.Datasec).Vars {
 				if vsi.Type.(*btf.Var).Name != rel.Name {
 					continue

elf_reader_test.go

@@ -750,6 +750,33 @@ func TestKfunc(t *testing.T) {
 	})
 }
 
+func TestWeakKfunc(t *testing.T) {
+	testutils.SkipOnOldKernel(t, "5.18", "kfunc support")
+	testutils.Files(t, testutils.Glob(t, "testdata/kfunc-e*.elf"), func(t *testing.T, file string) {
+		spec, err := LoadCollectionSpec(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if spec.ByteOrder != internal.NativeEndian {
+			return
+		}
+
+		var obj struct {
+			Missing *Program `ebpf:"weak_kfunc_missing"`
+			Calling *Program `ebpf:"call_weak_kfunc"`
+		}
+
+		err = spec.LoadAndAssign(&obj, nil)
+		testutils.SkipIfNotSupported(t, err)
+		if err != nil {
+			t.Fatalf("%+v", err)
+		}
+		defer obj.Missing.Close()
+		defer obj.Calling.Close()
+	})
+}
+
 func TestInvalidKfunc(t *testing.T) {
 	testutils.SkipOnOldKernel(t, "5.18", "kfunc support")
 

linker.go

@@ -1,6 +1,7 @@
 package ebpf
 
 import (
+	"debug/elf"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -259,6 +260,10 @@ func fixupAndValidate(insns asm.Instructions) error {
 	return nil
 }
 
+// POISON_CALL_KFUNC_BASE in libbpf.
+// https://github.com/libbpf/libbpf/blob/2778cbce609aa1e2747a69349f7f46a2f94f0522/src/libbpf.c#L5767
+const kfuncCallPoisonBase = 2002000000
+
 // fixupKfuncs loops over all instructions in search for kfunc calls.
 // If at least one is found, the current kernels BTF and module BTFis are searched to set Instruction.Constant
 // and Instruction.Offset to the correct values.
@@ -272,7 +277,7 @@ func fixupKfuncs(insns asm.Instructions) (_ handles, err error) {
 	iter := insns.Iterate()
 	for iter.Next() {
 		ins := iter.Ins
-		if ins.IsKfuncCall() {
+		if metadata := ins.Metadata.Get(kfuncMetaKey{}); metadata != nil {
 			goto fixups
 		}
 	}
@@ -292,7 +297,8 @@ fixups:
 	for {
 		ins := iter.Ins
 
-		if !ins.IsKfuncCall() {
+		metadata := ins.Metadata.Get(kfuncMetaKey{})
+		if metadata == nil {
 			if !iter.Next() {
 				// break loop if this was the last instruction in the stream.
 				break
@@ -301,15 +307,34 @@ fixups:
 		}
 
 		// check meta, if no meta return err
-		kfm, _ := ins.Metadata.Get(kfuncMeta{}).(*btf.Func)
+		kfm, _ := metadata.(*kfuncMeta)
 		if kfm == nil {
-			return nil, fmt.Errorf("kfunc call has no kfuncMeta")
+			return nil, fmt.Errorf("kfuncMetaKey doesn't contain kfuncMeta")
 		}
 
 		target := btf.Type((*btf.Func)(nil))
-		spec, module, err := findTargetInKernel(kernelSpec, kfm.Name, &target)
+		spec, module, err := findTargetInKernel(kernelSpec, kfm.Func.Name, &target)
+		if kfm.Binding == elf.STB_WEAK && errors.Is(err, btf.ErrNotFound) {
+			if ins.IsKfuncCall() {
+				// If the kfunc call is weak and not found, poison the call. Use a recognizable constant
+				// to make it easier to debug. And set src to zero so the verifier doesn't complain
+				// about the invalid imm/offset values before dead-code elimination.
+				ins.Constant = kfuncCallPoisonBase
+				ins.Src = 0
+			} else if ins.OpCode.IsDWordLoad() {
+				// If the kfunc DWordLoad is weak and not found, set its address to 0.
+				ins.Constant = 0
+				ins.Src = 0
+			} else {
+				return nil, fmt.Errorf("only kfunc calls and dword loads may have kfunc metadata")
+			}
+
+			iter.Next()
+			continue
+		}
+		// Error on non-weak kfunc not found.
 		if errors.Is(err, btf.ErrNotFound) {
-			return nil, fmt.Errorf("kfunc %q: %w", kfm.Name, ErrNotSupported)
+			return nil, fmt.Errorf("kfunc %q: %w", kfm.Func.Name, ErrNotSupported)
 		}
 		if err != nil {
 			return nil, err
@@ -320,8 +345,8 @@ fixups:
 			return nil, err
 		}
 
-		if err := btf.CheckTypeCompatibility(kfm.Type, target.(*btf.Func).Type); err != nil {
-			return nil, &incompatibleKfuncError{kfm.Name, err}
+		if err := btf.CheckTypeCompatibility(kfm.Func.Type, target.(*btf.Func).Type); err != nil {
+			return nil, &incompatibleKfuncError{kfm.Func.Name, err}
 		}
 
 		id, err := spec.TypeID(target)

testdata/common.h

@@ -20,6 +20,15 @@ enum libbpf_tristate {
 
 #define __kconfig __attribute__((section(".kconfig")))
 #define __ksym __attribute__((section(".ksyms")))
+#ifndef __weak
+#define __weak __attribute__((weak))
+#endif
+
+#define bpf_ksym_exists(sym) \
+	({ \
+		_Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \
+		!!sym; \
+	})
 
 #define BPF_MAP_TYPE_HASH (1)
 #define BPF_MAP_TYPE_ARRAY (2)

testdata/kfunc.c

@@ -8,6 +8,7 @@ struct __sk_buff {};
 struct nf_conn {};
 struct bpf_sock_tuple {};
 struct bpf_ct_opts {};
+struct bpf_cpumask {};
 
 extern struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *, struct bpf_sock_tuple *, uint32_t, struct bpf_ct_opts *, uint32_t) __ksym;
 extern void bpf_ct_release(struct nf_conn *) __ksym;
@@ -29,3 +30,29 @@ __section("fentry/bpf_fentry_test2") int benchmark() {
 	// the kernel when benchmarking the loader.
 	return bpf_fentry_test1(0);
 }
+
+extern void invalid_kfunc(void) __ksym __weak;
+
+extern struct bpf_cpumask *bpf_cpumask_create(void) __ksym __weak;
+extern void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym __weak;
+
+__section("tp_btf/task_newtask") int weak_kfunc_missing(void *ctx) {
+	if (bpf_ksym_exists(invalid_kfunc)) {
+		invalid_kfunc();
+		return 0;
+	}
+
+	return 1;
+}
+
+__section("tp_btf/task_newtask") int call_weak_kfunc(void *ctx) {
+	if (bpf_ksym_exists(bpf_cpumask_create)) {
+		struct bpf_cpumask *mask = bpf_cpumask_create();
+		if (mask)
+			bpf_cpumask_release(mask);
+
+		return 1;
+	}
+
+	return 0;
+}