diff --git a/.github/workflows/probe_load.yaml b/.github/workflows/probe_load.yaml new file mode 100644 index 000000000..1333da5f3 --- /dev/null +++ b/.github/workflows/probe_load.yaml @@ -0,0 +1,47 @@ +name: probe_load + +on: + push: + branches: [ main ] + pull_request: + +env: + go_version: '~1.22' + CGO_ENABLED: '0' + +jobs: + vm-test: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + tag: + - "stable" + - "6.6" + - "5.15" + - "5.10" + - "5.4" + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '${{ env.go_version }}' + - name: make docker-generate + run: make docker-generate + - name: verify output + run: make check-clean-work-tree + - name: Install vimto + run: go install lmb.io/vimto@latest + - name: Install qemu + run: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-system-x86 + sudo chmod 0666 /dev/kvm + - name: Test without verifier logs + id: no_verifier_logs_test + run: OTEL_GO_AUTO_SHOW_VERIFIER_LOG=false vimto -kernel :${{ matrix.tag }} -- go test -v -count=1 -tags=multi_kernel_test go.opentelemetry.io/auto/internal/pkg/instrumentation + - name: Test with verifier logs + run: OTEL_GO_AUTO_SHOW_VERIFIER_LOG=true vimto -kernel :${{ matrix.tag }} -- go test -v -count=1 -tags=multi_kernel_test go.opentelemetry.io/auto/internal/pkg/instrumentation + if: always() && steps.no_verifier_logs_test.outcome == 'failure' \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 80160748a..caee3f5bb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -74,6 +74,7 @@ linters-settings: files: - "!$test" - "!**/*test/*.go" + - "!**/testutils/*.go" deny: - pkg: "testing" - pkg: "github.com/stretchr/testify" diff --git a/.vimto.toml b/.vimto.toml new file mode 100644 index 000000000..8b82f6f51 --- /dev/null +++ b/.vimto.toml @@ -0,0 +1,12 @@ +kernel="ghcr.io/cilium/ci-kernels:stable" +smp="cpus=4" +memory="8G" +user="root" +setup=[ + "mount -t cgroup2 -o nosuid,noexec,nodev cgroup2 /sys/fs/cgroup", + "/bin/sh -c 'modprobe bpf_testmod || true'", + "dmesg --clear", +] +teardown=[ + "dmesg --read-clear", +] \ No newline at end of file diff --git a/internal/include/otel_types.h b/internal/include/otel_types.h index b62be270e..c8f92fd35 100644 --- a/internal/include/otel_types.h +++ b/internal/include/otel_types.h @@ -55,11 +55,6 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_ { u64 vtype = go_attr_value->vtype; - if (vtype == attr_type_invalid) { - bpf_printk("Invalid attribute value type\n"); - return false; - } - // Constant size values if (vtype == attr_type_bool || vtype == attr_type_int64 || @@ -74,7 +69,8 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_ bpf_printk("Aattribute string value is too long\n"); return false; } - return get_go_string_from_user_ptr(&go_attr_value->string, attr->value, OTEL_ATTRIBUTE_VALUE_MAX_LEN); + long res = bpf_probe_read_user(attr->value, go_attr_value->string.len & (OTEL_ATTRIBUTE_VALUE_MAX_LEN -1), go_attr_value->string.str); + return res == 0; } // TODO (#525): handle slices @@ -83,7 +79,7 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slice_len, otel_attributes_t *enc_attrs) { - if (attrs_buf == NULL || enc_attrs == NULL){ + if (attrs_buf == NULL){ return; } @@ -100,7 +96,10 @@ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slic return; } - for (u8 go_attr_index = 0; go_attr_index < num_attrs; go_attr_index++) { + for (u8 go_attr_index = 0; go_attr_index < OTEL_ATTRUBUTE_MAX_COUNT; go_attr_index++) { + if (go_attr_index >= slice_len) { + break; + } __builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t)); // Read the value struct bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value); @@ -124,9 +123,7 @@ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slic break; } - if (!get_go_string_from_user_ptr(&go_str, enc_attrs->attrs[valid_attrs].key, OTEL_ATTRIBUTE_KEY_MAX_LEN)) { - continue; - } + bpf_probe_read_user(enc_attrs->attrs[valid_attrs].key, go_str.len & (OTEL_ATTRIBUTE_KEY_MAX_LEN -1), go_str.str); if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) { continue; diff --git a/internal/pkg/inject/consts.go b/internal/pkg/inject/consts.go index 80a4955f0..53b9182d3 100644 --- a/internal/pkg/inject/consts.go +++ b/internal/pkg/inject/consts.go @@ -139,3 +139,7 @@ func WithOffset(key string, id structfield.ID, ver *version.Version) Option { } return WithKeyValue(key, off.Offset) } + +func GetLatestOffset(id structfield.ID) (structfield.OffsetKey, *version.Version) { + return offsets.GetLatestOffset(id) +} diff --git a/internal/pkg/instrumentation/manager.go b/internal/pkg/instrumentation/manager.go index 05b4c9a6b..e5c4382dd 100644 --- a/internal/pkg/instrumentation/manager.go +++ b/internal/pkg/instrumentation/manager.go @@ -223,21 +223,28 @@ func (m *Manager) cleanup(target *process.TargetDetails) error { return errors.Join(err, bpffsCleanup(target)) } -func (m *Manager) registerProbes() error { +//nolint:revive // ignoring linter complaint about control flag +func availableProbes(l logr.Logger, withTraceGlobal bool) []probe.Probe { insts := []probe.Probe{ - grpcClient.New(m.logger), - grpcServer.New(m.logger), - httpServer.New(m.logger), - httpClient.New(m.logger), - dbSql.New(m.logger), - kafkaProducer.New(m.logger), - kafkaConsumer.New(m.logger), + grpcClient.New(l), + grpcServer.New(l), + httpServer.New(l), + httpClient.New(l), + dbSql.New(l), + kafkaProducer.New(l), + kafkaConsumer.New(l), } - if m.globalImpl { - insts = append(insts, otelTraceGlobal.New(m.logger)) + if withTraceGlobal { + insts = append(insts, otelTraceGlobal.New(l)) } + return insts +} + +func (m *Manager) registerProbes() error { + insts := availableProbes(m.logger, m.globalImpl) + for _, i := range insts { err := m.registerProbe(i) if err != nil { diff --git a/internal/pkg/instrumentation/manager_load_test.go b/internal/pkg/instrumentation/manager_load_test.go new file mode 100644 index 000000000..68a1c338f --- /dev/null +++ b/internal/pkg/instrumentation/manager_load_test.go @@ -0,0 +1,57 @@ +//go:build multi_kernel_test + +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package instrumentation + +import ( + "log" + "os" + "testing" + + "github.com/go-logr/stdr" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/auto/internal/pkg/inject" + "go.opentelemetry.io/auto/internal/pkg/instrumentation/testutils" + "go.opentelemetry.io/auto/internal/pkg/instrumentation/utils" +) + +func TestLoadProbes(t *testing.T) { + ver, _ := utils.GetLinuxKernelVersion() + t.Logf("Running on kernel %s", ver.String()) + m := fakeManager(t) + + probes := availableProbes(m.logger, true) + assert.NotEmpty(t, probes) + + for _, p := range probes { + manifest := p.Manifest() + fields := manifest.StructFields + offsets := map[string]*version.Version{} + for _, f := range fields { + _, ver := inject.GetLatestOffset(f) + if ver != nil { + offsets[f.PkgPath] = ver + offsets[f.ModPath] = ver + } + } + t.Run(p.Manifest().Id.String(), func(t *testing.T) { + testProbe, ok := p.(testutils.TestProbe) + assert.True(t, ok) + testutils.ProbesLoad(t, testProbe, offsets) + }) + } +} + +func fakeManager(t *testing.T) *Manager { + logger := stdr.New(log.New(os.Stderr, "", log.LstdFlags)) + logger = logger.WithName("Instrumentation") + + m, err := NewManager(logger, nil, true, nil) + assert.NoError(t, err) + assert.NotNil(t, m) + + return m +} \ No newline at end of file diff --git a/internal/pkg/instrumentation/manager_test.go b/internal/pkg/instrumentation/manager_test.go index 0f9d6bf27..0fb624b4c 100644 --- a/internal/pkg/instrumentation/manager_test.go +++ b/internal/pkg/instrumentation/manager_test.go @@ -1,3 +1,5 @@ +//go:build !multi_kernel_test + // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 diff --git a/internal/pkg/instrumentation/probe/probe.go b/internal/pkg/instrumentation/probe/probe.go index 9b3e5856f..4a10894f7 100644 --- a/internal/pkg/instrumentation/probe/probe.go +++ b/internal/pkg/instrumentation/probe/probe.go @@ -92,6 +92,10 @@ func (i *Base[BPFObj, BPFEvent]) Manifest() Manifest { return NewManifest(i.ID, structfields, symbols) } +func (i *Base[BPFObj, BPFEvent]) Spec() (*ebpf.CollectionSpec, error) { + return i.SpecFn() +} + // Load loads all instrumentation offsets. func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetDetails) error { spec, err := i.SpecFn() @@ -99,7 +103,7 @@ func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetD return err } - err = i.injectConsts(td, spec) + err = i.InjectConsts(td, spec) if err != nil { return err } @@ -123,7 +127,7 @@ func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetD return nil } -func (i *Base[BPFObj, BPFEvent]) injectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error { +func (i *Base[BPFObj, BPFEvent]) InjectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error { opts, err := consts(i.Consts).injectOpts(td) if err != nil { return err diff --git a/internal/pkg/instrumentation/testutils/testutils.go b/internal/pkg/instrumentation/testutils/testutils.go new file mode 100644 index 000000000..f9498a8a2 --- /dev/null +++ b/internal/pkg/instrumentation/testutils/testutils.go @@ -0,0 +1,93 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package testutils + +import ( + "errors" + "testing" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/rlimit" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/assert" + + "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpffs" + "go.opentelemetry.io/auto/internal/pkg/instrumentation/utils" + "go.opentelemetry.io/auto/internal/pkg/process" +) + +var testGoVersion = version.Must(version.NewVersion("1.22.1")) + +type TestProbe interface { + Spec() (*ebpf.CollectionSpec, error) + InjectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error +} + +func ProbesLoad(t *testing.T, p TestProbe, libs map[string]*version.Version) { + err := rlimit.RemoveMemlock() + if !assert.NoError(t, err) { + return + } + + td := &process.TargetDetails{ + PID: 1, + AllocationDetails: &process.AllocationDetails{ + StartAddr: 140434497441792, + EndAddr: 140434497507328, + }, + Libraries: map[string]*version.Version{ + "std": testGoVersion, + }, + GoVersion: testGoVersion, + } + for k, v := range libs { + td.Libraries[k] = v + } + + err = bpffs.Mount(td) + if !assert.NoError(t, err) { + return + } + defer func() { + _ = bpffs.Cleanup(td) + }() + + spec, err := p.Spec() + if !assert.NoError(t, err) { + return + } + + // Inject the same constants as the BPF program. + // It is important to inject the same constants as those that will be used in the actual run, + // since From Linux 5.5 the verifier will use constants to eliminate dead code. + err = p.InjectConsts(td, spec) + if !assert.NoError(t, err) { + return + } + + opts := ebpf.CollectionOptions{ + Maps: ebpf.MapOptions{ + PinPath: bpffs.PathForTargetApplication(td), + }, + } + + collectVerifierLogs := utils.ShouldShowVerifierLogs() + if collectVerifierLogs { + opts.Programs.LogLevel = ebpf.LogLevelStats | ebpf.LogLevelInstruction + } + + c, err := ebpf.NewCollectionWithOptions(spec, opts) + if !assert.NoError(t, err) { + var ve *ebpf.VerifierError + if errors.As(err, &ve) && collectVerifierLogs { + t.Logf("Verifier log: %-100v\n", ve) + } + } + + defer func() { + if c != nil { + c.Close() + } + }() +} diff --git a/internal/pkg/instrumentation/utils/ebpf.go b/internal/pkg/instrumentation/utils/ebpf.go index 1db6788d1..b74f62bef 100644 --- a/internal/pkg/instrumentation/utils/ebpf.go +++ b/internal/pkg/instrumentation/utils/ebpf.go @@ -21,7 +21,7 @@ const ( // If the environment variable OTEL_GO_AUTO_SHOW_VERIFIER_LOG is set to true, the verifier log will be printed. func InitializeEBPFCollection(spec *ebpf.CollectionSpec, opts *ebpf.CollectionOptions) (*ebpf.Collection, error) { // Getting full verifier log is expensive, so we only do it if the user explicitly asks for it. - showVerifierLogs := shouldShowVerifierLogs() + showVerifierLogs := ShouldShowVerifierLogs() if showVerifierLogs { opts.Programs.LogLevel = ebpf.LogLevelInstruction | ebpf.LogLevelBranch | ebpf.LogLevelStats } @@ -37,8 +37,8 @@ func InitializeEBPFCollection(spec *ebpf.CollectionSpec, opts *ebpf.CollectionOp return c, err } -// shouldShowVerifierLogs returns if the user has configured verifier logs to be emitted. -func shouldShowVerifierLogs() bool { +// ShouldShowVerifierLogs returns if the user has configured verifier logs to be emitted. +func ShouldShowVerifierLogs() bool { val, exists := os.LookupEnv(showVerifierLogEnvVar) if exists { boolVal, err := strconv.ParseBool(val) diff --git a/internal/pkg/structfield/structfield.go b/internal/pkg/structfield/structfield.go index 5af844619..1ec511b1a 100644 --- a/internal/pkg/structfield/structfield.go +++ b/internal/pkg/structfield/structfield.go @@ -49,6 +49,20 @@ func (i *Index) GetOffset(id ID, ver *version.Version) (OffsetKey, bool) { return i.getOffset(id, ver) } +// GetLatestOffset returns the latest known offset value and version for id +// contained in the Index i. +func (i *Index) GetLatestOffset(id ID) (OffsetKey, *version.Version) { + i.dataMu.RLock() + defer i.dataMu.RUnlock() + + offs, ok := i.get(id) + if !ok { + return OffsetKey{}, nil + } + off, ver := offs.getLatest() + return off, ver.ToVersion() +} + func (i *Index) getOffset(id ID, ver *version.Version) (OffsetKey, bool) { offs, ok := i.get(id) if !ok { @@ -259,6 +273,23 @@ func (o *Offsets) Get(ver *version.Version) (OffsetKey, bool) { return v.offset, ok } +// getLatest returns the latest known offset value and version. +func (o *Offsets) getLatest() (OffsetKey, verKey) { + o.mu.RLock() + defer o.mu.RUnlock() + + latestVersion := verKey{} + val := OffsetKey{} + for verKey, ov := range o.values { + if verKey.GreaterThan(latestVersion) && ov.offset.Valid { + latestVersion = verKey + val = ov.offset + } + } + + return val, latestVersion +} + // Put sets the offset value for ver. If an offset for ver is already known // (i.e. ver.Equal(other) == true), this will overwrite that value. func (o *Offsets) Put(ver *version.Version, offset OffsetKey) { @@ -281,6 +312,28 @@ func (o *Offsets) Put(ver *version.Version, offset OffsetKey) { } } +func (v verKey) GreaterThan(other verKey) bool { + if v.major != other.major { + return v.major > other.major + } + if v.minor != other.minor { + return v.minor > other.minor + } + if v.patch != other.patch { + return v.patch > other.patch + } + return false +} + +func (v verKey) ToVersion() *version.Version { + vs := fmt.Sprintf("%d.%d.%d", v.major, v.minor, v.patch) + if v.prerelease != "" { + vs += "-" + v.prerelease + } + ver, _ := version.NewVersion(vs) + return ver +} + func (o *Offsets) index() map[OffsetKey][]*version.Version { o.mu.RLock() defer o.mu.RUnlock() diff --git a/internal/pkg/structfield/structfield_test.go b/internal/pkg/structfield/structfield_test.go index daa7dbfd6..4b36e78b0 100644 --- a/internal/pkg/structfield/structfield_test.go +++ b/internal/pkg/structfield/structfield_test.go @@ -53,6 +53,10 @@ func TestOffsets(t *testing.T) { assert.True(t, ok, "did not get 1.2.1") assert.Equal(t, OffsetKey{Offset: 2, Valid: true}, off, "invalid value for 1.2.1") + off, ver := o.getLatest() + assert.Equal(t, v121, ver.ToVersion(), "invalid version for latest") + assert.Equal(t, OffsetKey{Offset: 2, Valid: true}, off, "invalid value for latest") + o.Put(v120, OffsetKey{Offset: 1, Valid: true}) off, ok = o.Get(v120) assert.True(t, ok, "did not get 1.2.0 after reset") @@ -123,3 +127,21 @@ func TestIndexUnmarshalJSON(t *testing.T) { require.NoError(t, json.NewDecoder(f).Decode(&got)) assert.Equal(t, index, &got) } + +func TestGetLatestOffsetFromIndex(t *testing.T) { + off, ver := index.GetLatestOffset(NewID("std", "net/http", "Request", "Method")) + assert.Equal(t, v130, ver, "invalid version for Request.Method") + assert.Equal(t, OffsetKey{Offset: 1, Valid: true}, off, "invalid value for Request.Method") + + off, ver = index.GetLatestOffset(NewID("std", "net/http", "Request", "URL")) + assert.Equal(t, v130, ver, "invalid version for Request.URL") + assert.Equal(t, OffsetKey{Offset: 2, Valid: true}, off, "invalid value for Request.URL") + + off, ver = index.GetLatestOffset(NewID("std", "net/http", "Response", "Status")) + assert.Equal(t, v120, ver, "invalid version for Response.Status") + assert.Equal(t, OffsetKey{Offset: 0, Valid: true}, off, "invalid value for Response.Status") + + off, ver = index.GetLatestOffset(NewID("google.golang.org/grpc", "google.golang.org/grpc", "ClientConn", "target")) + assert.Equal(t, v120, ver, "invalid version for ClientConn.target") + assert.Equal(t, OffsetKey{Offset: 0, Valid: true}, off, "invalid value for ClientConn.target") +}