Add probe loading tests across multiple kernel versions using vimto (#…

…1005)

.github/workflows/probe_load.yaml

@@ -0,0 +1,47 @@
+name: probe_load
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+env:
+  go_version: '~1.22'
+  CGO_ENABLED: '0'
+
+jobs:
+    vm-test:
+        name: Run tests
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+        strategy:
+          fail-fast: false
+          matrix:
+            tag:
+              - "stable"
+              - "6.6"
+              - "5.15"
+              - "5.10"
+              - "5.4"
+        steps:
+          - uses: actions/checkout@v4
+          - name: Set up Go
+            uses: actions/setup-go@v5
+            with:
+              go-version: '${{ env.go_version }}'
+          - name: make docker-generate
+            run: make docker-generate
+          - name: verify output
+            run: make check-clean-work-tree
+          - name: Install vimto
+            run: go install lmb.io/vimto@latest
+          - name: Install qemu
+            run: |
+                sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-system-x86
+                sudo chmod 0666 /dev/kvm
+          - name: Test without verifier logs
+            id: no_verifier_logs_test
+            run: OTEL_GO_AUTO_SHOW_VERIFIER_LOG=false vimto -kernel :${{ matrix.tag }} -- go test -v -count=1 -tags=multi_kernel_test go.opentelemetry.io/auto/internal/pkg/instrumentation
+          - name: Test with verifier logs
+            run: OTEL_GO_AUTO_SHOW_VERIFIER_LOG=true vimto -kernel :${{ matrix.tag }} -- go test -v -count=1 -tags=multi_kernel_test go.opentelemetry.io/auto/internal/pkg/instrumentation
+            if: always() && steps.no_verifier_logs_test.outcome == 'failure'

.golangci.yml

@@ -74,6 +74,7 @@ linters-settings:
         files:
           - "!$test"
           - "!**/*test/*.go"
+          - "!**/testutils/*.go"
         deny:
           - pkg: "testing"
           - pkg: "github.com/stretchr/testify"

.vimto.toml

@@ -0,0 +1,12 @@
+kernel="ghcr.io/cilium/ci-kernels:stable"
+smp="cpus=4"
+memory="8G"
+user="root"
+setup=[
+   "mount -t cgroup2 -o nosuid,noexec,nodev cgroup2 /sys/fs/cgroup",
+   "/bin/sh -c 'modprobe bpf_testmod || true'",
+   "dmesg --clear",
+]
+teardown=[
+   "dmesg --read-clear",
+]

internal/include/otel_types.h

@@ -55,11 +55,6 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_
 {
 	u64 vtype = go_attr_value->vtype;
 
-	if (vtype == attr_type_invalid) {
-		bpf_printk("Invalid attribute value type\n");
-		return false;
-	}
-
 	// Constant size values
 	if (vtype == attr_type_bool ||
 		vtype == attr_type_int64 ||
@@ -74,7 +69,8 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_
 			bpf_printk("Aattribute string value is too long\n");
 			return false;
 		}
-		return get_go_string_from_user_ptr(&go_attr_value->string, attr->value, OTEL_ATTRIBUTE_VALUE_MAX_LEN);
+		long res = bpf_probe_read_user(attr->value, go_attr_value->string.len & (OTEL_ATTRIBUTE_VALUE_MAX_LEN -1), go_attr_value->string.str);
+ 		return res == 0;
 	}
 
 	// TODO (#525): handle slices
@@ -83,7 +79,7 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_
 
 static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slice_len, otel_attributes_t *enc_attrs)
 {
-	if (attrs_buf == NULL || enc_attrs == NULL){
+	if (attrs_buf == NULL){
 		return;
 	}
 
@@ -100,7 +96,10 @@ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slic
 		return;
 	}
 
-	for (u8 go_attr_index = 0; go_attr_index < num_attrs; go_attr_index++) {
+	for (u8 go_attr_index = 0; go_attr_index < OTEL_ATTRUBUTE_MAX_COUNT; go_attr_index++) {
+		if (go_attr_index >= slice_len) {
+ 			break;
+ 		}
 		__builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t));
 		// Read the value struct
 		bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value);
@@ -124,9 +123,7 @@ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slic
 			break;
 		}
 
-		if (!get_go_string_from_user_ptr(&go_str, enc_attrs->attrs[valid_attrs].key, OTEL_ATTRIBUTE_KEY_MAX_LEN)) {
-			continue;
-		}
+		bpf_probe_read_user(enc_attrs->attrs[valid_attrs].key, go_str.len & (OTEL_ATTRIBUTE_KEY_MAX_LEN -1), go_str.str);
 
 		if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) {
 			continue;

internal/pkg/inject/consts.go

@@ -139,3 +139,7 @@ func WithOffset(key string, id structfield.ID, ver *version.Version) Option {
 	}
 	return WithKeyValue(key, off.Offset)
 }
+
+func GetLatestOffset(id structfield.ID) (structfield.OffsetKey, *version.Version) {
+	return offsets.GetLatestOffset(id)
+}

internal/pkg/instrumentation/manager.go

@@ -223,21 +223,28 @@ func (m *Manager) cleanup(target *process.TargetDetails) error {
 	return errors.Join(err, bpffsCleanup(target))
 }
 
-func (m *Manager) registerProbes() error {
+//nolint:revive // ignoring linter complaint about control flag
+func availableProbes(l logr.Logger, withTraceGlobal bool) []probe.Probe {
 	insts := []probe.Probe{
-		grpcClient.New(m.logger),
-		grpcServer.New(m.logger),
-		httpServer.New(m.logger),
-		httpClient.New(m.logger),
-		dbSql.New(m.logger),
-		kafkaProducer.New(m.logger),
-		kafkaConsumer.New(m.logger),
+		grpcClient.New(l),
+		grpcServer.New(l),
+		httpServer.New(l),
+		httpClient.New(l),
+		dbSql.New(l),
+		kafkaProducer.New(l),
+		kafkaConsumer.New(l),
 	}
 
-	if m.globalImpl {
-		insts = append(insts, otelTraceGlobal.New(m.logger))
+	if withTraceGlobal {
+		insts = append(insts, otelTraceGlobal.New(l))
 	}
 
+	return insts
+}
+
+func (m *Manager) registerProbes() error {
+	insts := availableProbes(m.logger, m.globalImpl)
+
 	for _, i := range insts {
 		err := m.registerProbe(i)
 		if err != nil {

internal/pkg/instrumentation/manager_load_test.go

@@ -0,0 +1,57 @@
+//go:build multi_kernel_test
+
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package instrumentation
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/go-logr/stdr"
+	"github.com/hashicorp/go-version"
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/auto/internal/pkg/inject"
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/testutils"
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/utils"
+)
+
+func TestLoadProbes(t *testing.T) {
+	ver, _ := utils.GetLinuxKernelVersion()
+	t.Logf("Running on kernel %s", ver.String())
+	m := fakeManager(t)
+
+	probes := availableProbes(m.logger, true)
+	assert.NotEmpty(t, probes)
+
+	for _, p := range probes {
+		manifest := p.Manifest()
+		fields := manifest.StructFields
+		offsets := map[string]*version.Version{}
+		for _, f := range fields {
+			_, ver := inject.GetLatestOffset(f)
+			if ver != nil {
+				offsets[f.PkgPath] = ver
+				offsets[f.ModPath] = ver
+			}
+		}
+		t.Run(p.Manifest().Id.String(), func(t *testing.T) {
+			testProbe, ok := p.(testutils.TestProbe)
+			assert.True(t, ok)
+			testutils.ProbesLoad(t, testProbe, offsets)
+		})
+	}
+}
+
+func fakeManager(t *testing.T) *Manager {
+	logger := stdr.New(log.New(os.Stderr, "", log.LstdFlags))
+	logger = logger.WithName("Instrumentation")
+
+	m, err := NewManager(logger, nil, true, nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, m)
+
+	return m
+} 

internal/pkg/instrumentation/manager_test.go

@@ -1,3 +1,5 @@
+//go:build !multi_kernel_test
+
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 

internal/pkg/instrumentation/probe/probe.go

@@ -92,14 +92,18 @@ func (i *Base[BPFObj, BPFEvent]) Manifest() Manifest {
 	return NewManifest(i.ID, structfields, symbols)
 }
 
+func (i *Base[BPFObj, BPFEvent]) Spec() (*ebpf.CollectionSpec, error) {
+	return i.SpecFn()
+}
+
 // Load loads all instrumentation offsets.
 func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetDetails) error {
 	spec, err := i.SpecFn()
 	if err != nil {
 		return err
 	}
 
-	err = i.injectConsts(td, spec)
+	err = i.InjectConsts(td, spec)
 	if err != nil {
 		return err
 	}
@@ -123,7 +127,7 @@ func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetD
 	return nil
 }
 
-func (i *Base[BPFObj, BPFEvent]) injectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error {
+func (i *Base[BPFObj, BPFEvent]) InjectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error {
 	opts, err := consts(i.Consts).injectOpts(td)
 	if err != nil {
 		return err

internal/pkg/instrumentation/testutils/testutils.go

@@ -0,0 +1,93 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package testutils
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/rlimit"
+	"github.com/hashicorp/go-version"
+	"github.com/stretchr/testify/assert"
+
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/bpffs"
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/utils"
+	"go.opentelemetry.io/auto/internal/pkg/process"
+)
+
+var testGoVersion = version.Must(version.NewVersion("1.22.1"))
+
+type TestProbe interface {
+	Spec() (*ebpf.CollectionSpec, error)
+	InjectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error
+}
+
+func ProbesLoad(t *testing.T, p TestProbe, libs map[string]*version.Version) {
+	err := rlimit.RemoveMemlock()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	td := &process.TargetDetails{
+		PID: 1,
+		AllocationDetails: &process.AllocationDetails{
+			StartAddr: 140434497441792,
+			EndAddr:   140434497507328,
+		},
+		Libraries: map[string]*version.Version{
+			"std": testGoVersion,
+		},
+		GoVersion: testGoVersion,
+	}
+	for k, v := range libs {
+		td.Libraries[k] = v
+	}
+
+	err = bpffs.Mount(td)
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer func() {
+		_ = bpffs.Cleanup(td)
+	}()
+
+	spec, err := p.Spec()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// Inject the same constants as the BPF program.
+	// It is important to inject the same constants as those that will be used in the actual run,
+	// since From Linux 5.5 the verifier will use constants to eliminate dead code.
+	err = p.InjectConsts(td, spec)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	opts := ebpf.CollectionOptions{
+		Maps: ebpf.MapOptions{
+			PinPath: bpffs.PathForTargetApplication(td),
+		},
+	}
+
+	collectVerifierLogs := utils.ShouldShowVerifierLogs()
+	if collectVerifierLogs {
+		opts.Programs.LogLevel = ebpf.LogLevelStats | ebpf.LogLevelInstruction
+	}
+
+	c, err := ebpf.NewCollectionWithOptions(spec, opts)
+	if !assert.NoError(t, err) {
+		var ve *ebpf.VerifierError
+		if errors.As(err, &ve) && collectVerifierLogs {
+			t.Logf("Verifier log: %-100v\n", ve)
+		}
+	}
+
+	defer func() {
+		if c != nil {
+			c.Close()
+		}
+	}()
+}

internal/pkg/instrumentation/utils/ebpf.go

@@ -21,7 +21,7 @@ const (
 // If the environment variable OTEL_GO_AUTO_SHOW_VERIFIER_LOG is set to true, the verifier log will be printed.
 func InitializeEBPFCollection(spec *ebpf.CollectionSpec, opts *ebpf.CollectionOptions) (*ebpf.Collection, error) {
 	// Getting full verifier log is expensive, so we only do it if the user explicitly asks for it.
-	showVerifierLogs := shouldShowVerifierLogs()
+	showVerifierLogs := ShouldShowVerifierLogs()
 	if showVerifierLogs {
 		opts.Programs.LogLevel = ebpf.LogLevelInstruction | ebpf.LogLevelBranch | ebpf.LogLevelStats
 	}
@@ -37,8 +37,8 @@ func InitializeEBPFCollection(spec *ebpf.CollectionSpec, opts *ebpf.CollectionOp
 	return c, err
 }
 
-// shouldShowVerifierLogs returns if the user has configured verifier logs to be emitted.
-func shouldShowVerifierLogs() bool {
+// ShouldShowVerifierLogs returns if the user has configured verifier logs to be emitted.
+func ShouldShowVerifierLogs() bool {
 	val, exists := os.LookupEnv(showVerifierLogEnvVar)
 	if exists {
 		boolVal, err := strconv.ParseBool(val)