link: implement uprobe_multi link type

This commit adds support for this through the UprobeMulti() and
UretprobeMulti() APIs in package link.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>

internal/unix/types_linux.go

@@ -40,6 +40,7 @@ const (
 	BPF_F_MMAPABLE            = linux.BPF_F_MMAPABLE
 	BPF_F_INNER_MAP           = linux.BPF_F_INNER_MAP
 	BPF_F_KPROBE_MULTI_RETURN = linux.BPF_F_KPROBE_MULTI_RETURN
+	BPF_F_UPROBE_MULTI_RETURN = linux.BPF_F_UPROBE_MULTI_RETURN
 	BPF_OBJ_NAME_LEN          = linux.BPF_OBJ_NAME_LEN
 	BPF_TAG_SIZE              = linux.BPF_TAG_SIZE
 	BPF_RINGBUF_BUSY_BIT      = linux.BPF_RINGBUF_BUSY_BIT

internal/unix/types_other.go

@@ -42,6 +42,7 @@ const (
 	BPF_F_MMAPABLE
 	BPF_F_INNER_MAP
 	BPF_F_KPROBE_MULTI_RETURN
+	BPF_F_UPROBE_MULTI_RETURN
 	BPF_F_XDP_HAS_FRAGS
 	BPF_OBJ_NAME_LEN
 	BPF_TAG_SIZE

link/link.go

@@ -96,6 +96,8 @@ func wrapRawLink(raw *RawLink) (_ Link, err error) {
 		return &NetNsLink{*raw}, nil
 	case KprobeMultiType:
 		return &kprobeMultiLink{*raw}, nil
+	case UprobeMultiType:
+		return &uprobeMultiLink{*raw}, nil
 	case PerfEventType:
 		return nil, fmt.Errorf("recovering perf event fd: %w", ErrNotSupported)
 	case TCXType:
@@ -316,7 +318,7 @@ func (l *RawLink) Info() (*Info, error) {
 	case XDPType:
 		extra = &XDPInfo{}
 	case RawTracepointType, IterType,
-		PerfEventType, KprobeMultiType:
+		PerfEventType, KprobeMultiType, UprobeMultiType:
 		// Extra metadata not supported.
 	case TCXType:
 		extra = &TCXInfo{}

link/syscalls.go

@@ -25,6 +25,7 @@ const (
 	PerfEventType     = sys.BPF_LINK_TYPE_PERF_EVENT
 	KprobeMultiType   = sys.BPF_LINK_TYPE_KPROBE_MULTI
 	TCXType           = sys.BPF_LINK_TYPE_TCX
+	UprobeMultiType   = sys.BPF_LINK_TYPE_UPROBE_MULTI
 )
 
 var haveProgAttach = internal.NewFeatureTest("BPF_PROG_ATTACH", "4.10", func() error {

link/uprobe.go

@@ -159,6 +159,48 @@ func (ex *Executable) load(f *internal.SafeELFFile) error {
 	return nil
 }
 
+// address calculates the address of a symbol in the executable.
+//
+// opts must not be nil.
+func (ex *Executable) addressMulti(symbols []string) ([]uint64, error) {
+	var err error
+	ex.addressesOnce.Do(func() {
+		var f *internal.SafeELFFile
+		f, err = internal.OpenSafeELFFile(ex.path)
+		if err != nil {
+			err = fmt.Errorf("parse ELF file: %w", err)
+			return
+		}
+		defer f.Close()
+
+		err = ex.load(f)
+	})
+	if err != nil {
+		return []uint64{0}, fmt.Errorf("lazy load symbols: %w", err)
+	}
+
+	var addresses []uint64
+	for _, symbol := range symbols {
+		address, ok := ex.addresses[symbol]
+		if !ok {
+			return []uint64{0}, fmt.Errorf("symbol %s: %w", symbol, ErrNoSymbol)
+		}
+
+		// Symbols with location 0 from section undef are shared library calls and
+		// are relocated before the binary is executed. Dynamic linking is not
+		// implemented by the library, so mark this as unsupported for now.
+		//
+		// Since only offset values are stored and not elf.Symbol, if the value is 0,
+		// assume it's an external symbol.
+		if address == 0 {
+			return []uint64{0}, fmt.Errorf("cannot resolve %s library call '%s': %w "+
+				"(consider providing UprobeOptions.Address)", ex.path, symbol, ErrNotSupported)
+		}
+		addresses = append(addresses, address)
+	}
+	return addresses, nil
+}
+
 // address calculates the address of a symbol in the executable.
 //
 // opts must not be nil.

link/uprobe_multi.go

@@ -0,0 +1,148 @@
+package link
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"unsafe"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/asm"
+	"github.com/cilium/ebpf/internal"
+	"github.com/cilium/ebpf/internal/sys"
+	"github.com/cilium/ebpf/internal/unix"
+)
+
+type UprobeMultiOptions struct {
+	Path          string
+	Offsets       []uint64
+	RefCtrOffsets []uint64
+	Cookies       []uint64
+}
+
+func (ex *Executable) UprobeMulti(symbols []string, prog *ebpf.Program, opts *UprobeMultiOptions) (Link, error) {
+	if opts == nil {
+		opts = &UprobeMultiOptions{}
+	}
+	var err error
+	if len(symbols) != 0 {
+		opts.Offsets, err = ex.addressMulti(symbols)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return ex.uprobeMulti(prog, opts, 0)
+}
+
+func (ex *Executable) UretprobeMulti(symbols []string, prog *ebpf.Program, opts *UprobeMultiOptions) (Link, error) {
+	return ex.uprobeMulti(prog, opts, unix.BPF_F_UPROBE_MULTI_RETURN)
+}
+
+func (ex *Executable) uprobeMulti(prog *ebpf.Program, opts *UprobeMultiOptions, flags uint32) (Link, error) {
+	if prog == nil {
+		return nil, errors.New("cannot attach a nil program")
+	}
+
+	offsets := uint32(len(opts.Offsets))
+	refCtrOffsets := uint32(len(opts.RefCtrOffsets))
+	cookies := uint32(len(opts.Cookies))
+
+	if offsets == 0 {
+		return nil, fmt.Errorf("Offsets is required: %w", errInvalidInput)
+	}
+	if refCtrOffsets > 0 && refCtrOffsets != offsets {
+		return nil, fmt.Errorf("RefCtrOffsets must be exactly Offsets in length: %w", errInvalidInput)
+	}
+	if cookies > 0 && cookies != offsets {
+		return nil, fmt.Errorf("Cookies must be exactly Offsets in length: %w", errInvalidInput)
+	}
+
+	attr := &sys.LinkCreateUprobeMultiAttr{
+		Path:             sys.NewStringPointer(ex.path),
+		ProgFd:           uint32(prog.FD()),
+		AttachType:       sys.BPF_TRACE_UPROBE_MULTI,
+		UprobeMultiFlags: flags,
+		Count:            offsets,
+		Offsets:          sys.NewPointer(unsafe.Pointer(&opts.Offsets[0])),
+	}
+
+	if refCtrOffsets != 0 {
+		attr.RefCtrOffsets = sys.NewPointer(unsafe.Pointer(&opts.RefCtrOffsets[0]))
+	}
+	if cookies != 0 {
+		attr.Cookies = sys.NewPointer(unsafe.Pointer(&opts.Cookies[0]))
+	}
+
+	fd, err := sys.LinkCreateUprobeMulti(attr)
+	if errors.Is(err, unix.ESRCH) {
+		return nil, fmt.Errorf("XXX: %w", os.ErrNotExist)
+	}
+	if errors.Is(err, unix.EINVAL) {
+		return nil, fmt.Errorf("%w (missing kernel symbol or prog's AttachType not AttachTraceUprobeMulti?)", err)
+	}
+
+	if err != nil {
+		if haveFeatErr := haveBPFLinkUprobeMulti(); haveFeatErr != nil {
+			return nil, haveFeatErr
+		}
+		return nil, err
+	}
+
+	return &uprobeMultiLink{RawLink{fd, ""}}, nil
+}
+
+type uprobeMultiLink struct {
+	RawLink
+}
+
+var _ Link = (*uprobeMultiLink)(nil)
+
+func (kml *uprobeMultiLink) Update(prog *ebpf.Program) error {
+	return fmt.Errorf("update uprobe_multi: %w", ErrNotSupported)
+}
+
+func (kml *uprobeMultiLink) Pin(string) error {
+	return fmt.Errorf("pin uprobe_multi: %w", ErrNotSupported)
+}
+
+func (kml *uprobeMultiLink) Unpin() error {
+	return fmt.Errorf("unpin uprobe_multi: %w", ErrNotSupported)
+}
+
+var haveBPFLinkUprobeMulti = internal.NewFeatureTest("bpf_link_uprobe_multi", "6.6", func() error {
+	prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{
+		Name: "probe_upm_link",
+		Type: ebpf.Kprobe,
+		Instructions: asm.Instructions{
+			asm.Mov.Imm(asm.R0, 0),
+			asm.Return(),
+		},
+		AttachType: ebpf.AttachTraceUprobeMulti,
+		License:    "MIT",
+	})
+	if errors.Is(err, unix.E2BIG) {
+		// Kernel doesn't support AttachType field.
+		return internal.ErrNotSupported
+	}
+	if err != nil {
+		return err
+	}
+	defer prog.Close()
+
+	fd, err := sys.LinkCreateUprobeMulti(&sys.LinkCreateUprobeMultiAttr{
+		ProgFd:     uint32(prog.FD()),
+		AttachType: sys.BPF_TRACE_UPROBE_MULTI,
+		Path:       sys.NewStringPointer("/"),
+		Offsets:    sys.NewPointer(unsafe.Pointer(&[]uint64{0})),
+		Count:      1,
+	})
+	switch {
+	case errors.Is(err, unix.EBADF):
+		return nil
+	case err != nil:
+		return internal.ErrNotSupported
+	}
+	// should not happen
+	fd.Close()
+	return internal.ErrNotSupported
+})

link/uprobe_multi_test.go

@@ -0,0 +1,133 @@
+package link
+
+import (
+	"errors"
+	"os/exec"
+	"testing"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/internal/testutils"
+	"github.com/cilium/ebpf/internal/unix"
+)
+
+func TestUprobeMulti(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkUprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceUprobeMulti, "")
+
+	um, err := bashEx.UprobeMulti([]string{"bash_logout", "main"}, prog, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer um.Close()
+
+	testLink(t, um, prog)
+}
+
+func TestUprobeMultiInput(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkUprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceUprobeMulti, "")
+
+	// One of Symbols or Addresses must be given.
+	_, err := bashEx.UprobeMulti([]string{}, prog, nil)
+	if !errors.Is(err, errInvalidInput) {
+		t.Fatalf("expected errInvalidInput, got: %v", err)
+	}
+
+	// One Symbol, two cookies..
+	_, err = bashEx.UprobeMulti([]string{}, prog, &UprobeMultiOptions{
+		Offsets: []uint64{1},
+		Cookies: []uint64{2, 3},
+	})
+	if !errors.Is(err, errInvalidInput) {
+		t.Fatalf("expected errInvalidInput, got: %v", err)
+	}
+}
+
+func TestUprobeMultiErrors(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkUprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceUprobeMulti, "")
+
+	// Wrong offset
+	um, err := bashEx.UprobeMulti([]string{}, prog, &UprobeMultiOptions{Offsets: []uint64{1<<64 - 1}})
+	if !errors.Is(err, unix.EINVAL) {
+		um.Close()
+		t.Skipf("need kernel change, skipping for now, expected EINVAL, got: %s", err)
+	}
+}
+
+func TestUprobeMultiCookie(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkUprobeMulti())
+
+	prog := mustLoadProgram(t, ebpf.Kprobe, ebpf.AttachTraceUprobeMulti, "")
+
+	um, err := bashEx.UprobeMulti([]string{"bash_logout", "main"}, prog,
+		&UprobeMultiOptions{
+			Cookies: []uint64{1, 2},
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = um.Close()
+}
+
+func TestUprobeMultiProgramCall(t *testing.T) {
+	testutils.SkipIfNotSupported(t, haveBPFLinkUprobeMulti())
+
+	args := []string{"--help"}
+	elf := "/bin/bash"
+
+	m, p := newUpdaterMapProg(t, ebpf.Kprobe, ebpf.AttachTraceUprobeMulti)
+
+	// Load the executable.
+	ex, err := OpenExecutable(elf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Open Uprobe on the executable for the given symbol
+	// and attach it to the ebpf program created above.
+	um, err := ex.UprobeMulti([]string{"main", "_start"}, p, nil)
+	if errors.Is(err, ErrNoSymbol) {
+		// Assume bash::main and go::main.main always exists
+		// and skip the test if the symbol can't be found as
+		// certain OS (eg. Debian) strip binaries.
+		t.Skipf("executable %s appear to be stripped, skipping", elf)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Trigger ebpf program call.
+	trigger := func(t *testing.T) {
+		if err := exec.Command(elf, args...).Run(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	trigger(t)
+
+	// Assert that the value at index 0 has been updated to 1.
+	assertMapValue(t, m, 0, 1)
+
+	// Detach the Uprobe.
+	if err := um.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Reset map value to 0 at index 0.
+	if err := m.Update(uint32(0), uint32(0), ebpf.UpdateExist); err != nil {
+		t.Fatal(err)
+	}
+
+	// Retrigger the ebpf program call.
+	trigger(t)
+
+	// Assert that this time the value has not been updated.
+	assertMapValue(t, m, 0, 0)
+}
+
+func TestHaveBPFLinkUprobeMulti(t *testing.T) {
+	testutils.CheckFeatureTest(t, haveBPFLinkUprobeMulti)
+}