Merge branch 'main' into sblumenthal/fix-missing-k8s-events

.github/CODEOWNERS

@@ -592,6 +592,7 @@
 /test/new-e2e/tests/npm                       @DataDog/Networks
 /test/new-e2e/tests/npm/ec2_1host_wkit_test.go @DataDog/Networks @DataDog/windows-kernel-integrations
 /test/new-e2e/tests/orchestrator              @DataDog/container-app
+/test/new-e2e/tests/otel                      @DataDog/opentelemetry
 /test/new-e2e/tests/process                   @DataDog/processes
 /test/new-e2e/tests/sysprobe-functional     @DataDog/windows-kernel-integrations
 /test/new-e2e/tests/security-agent-functional     @DataDog/windows-kernel-integrations

.gitlab-ci.yml

@@ -821,6 +821,7 @@ workflow:
         - comp/core/autodiscovery/listeners/**/*
         - comp/core/autodiscovery/providers/**/*
         - comp/languagedetection/**/*
+        - pkg/clusteragent/admission/mutate/**/*
         - pkg/collector/corechecks/cluster/**/*
         - pkg/collector/corechecks/containers/**/*
         - pkg/collector/corechecks/containerimage/**/*
@@ -986,6 +987,15 @@ workflow:
   - when: manual
     allow_failure: true
 
+.on_otel_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - comp/otelcol/**/*
+        - test/new-e2e/tests/otel/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+
 .on_windows_service_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 3c7d2dc2d3dd
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 94712f9a273b

.gitlab/e2e/e2e.yml

@@ -405,6 +405,19 @@ new-e2e-ndm-snmp:
     TARGETS: ./tests/ndm/snmp
     TEAM: network-device-monitoring
 
+new-e2e-otel:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_otel_or_e2e_changes]
+    - !reference [.manual]
+  needs:
+    - !reference [.needs_new_e2e_template]
+    - qa_dca
+    - qa_agent
+  variables:
+    TARGETS: ./tests/otel
+    TEAM: otel
+
 generate-flakes-finder-pipeline:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   stage: e2e

LICENSE-3rdparty.csv

@@ -2692,7 +2692,7 @@ core,golang.org/x/text/unicode/bidi,BSD-3-Clause,Copyright (c) 2009 The Go Autho
 core,golang.org/x/text/unicode/norm,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/text/width,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 core,golang.org/x/time/rate,BSD-3-Clause,Copyright 2009 The Go Authors
-core,golang/go,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved.
+core,golang/go,BSD-Source-Code,Copyright (c) 2009 The Go Authors. All rights reserved.
 core,gomodules.xyz/jsonpatch/v2,Apache-2.0,Copyright (c) 2015 The Authors
 core,gonum.org/v1/gonum/blas,BSD-3-Clause,Alexander Egurnov <alexander.egurnov@gmail.com> | Andrei Blinnikov <goofinator@mail.ru> | Andrew Brampton <brampton@gmail.com> | Bailey Lissington <lissington4@gmail.com> | Bill Gray <wgray@gogray.com> | Bill Noon <noon.bill@gmail.com> | Brendan Tracey <tracey.brendan@gmail.com> | Brent Pedersen <bpederse@gmail.com> | Bulat Khasanov <afti@yandex.ru> | Chad Kunde <kunde21@gmail.com> | Chan Kwan Yin <sofe2038@gmail.com> | Chih-Wei Chang <bert.cwchang@gmail.com> | Chong-Yeol Nah <nahchongyeol@gmail.com> | Chris Tessum <ctessum@gmail.com> | Christophe Meessen <christophe.meessen@gmail.com> | Christopher Waldon <christopher.waldon.dev@gmail.com> | Clayton Northey <clayton.northey@gmail.com> | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak <dan.kortschak@adelaide.edu.au> <dan@kortschak.io> | Dan Lorenc <lorenc.d@gmail.com> | Daniel Fireman <danielfireman@gmail.com> | Dario Heinisch <dario.heinisch@gmail.com> | David Kleiven <davidkleiven446@gmail.com> | David Samborski <bloggingarrow@gmail.com> | Davor Kapsa <davor.kapsa@gmail.com> | DeepMind Technologies | Delaney Gillilan <delaneygillilan@gmail.com> | Dezmond Goff <goff.dezmond@gmail.com> | Dong-hee Na <donghee.na92@gmail.com> | Dustin Spicuzza <dustin@virtualroadside.com> | Egon Elbre <egonelbre@gmail.com> | Ekaterina Efimova <katerina.efimova@gmail.com> | Eng Zer Jun <engzerjun@gmail.com> | Ethan Burns <burns.ethan@gmail.com> | Ethan Reesor <ethan.reesor@gmail.com> | Evert Lammerts <evert.lammerts@gmail.com> | Evgeny Savinov <notime.sea@gmail.com> | Fabian Wickborn <fabian@wickborn.net> | Facundo Gaich <facugaich@gmail.com> | Fazlul Shahriar <fshahriar@gmail.com> | Francesc Campoy <campoy@golang.org> | Google Inc | Gustaf Johansson <gustaf@pinon.se> | Hossein Zolfi <hossein.zolfi@gmail.com> | Iakov Davydov <iakov.davydov@unil.ch> | Igor Mikushkin <igor.mikushkin@gmail.com> | Iskander Sharipov <quasilyte@gmail.com> | Jalem Raj Rohit <jrajrohit33@gmail.com> | James Bell <james@stellentus.com> | James Bowman <james.edward.bowman@gmail.com> | James Holmes <32bitkid@gmail.com> | Janne Snabb <snabb@epipe.com> | Jeremy Atkinson <jchatkinson@gmail.com> | Jes Cok <xigua67damn@gmail.com> | Jinesi Yelizati <i63888888@163.com> | Jon Richards <noj.richards@gmail.com> | Jonas Kahler <jonas@derkahler.de> | Jonas Schulze <jonas.schulze@ovgu.de> | Jonathan Bluett-Duncan <jbluettduncan@gmail.com> | Jonathan J Lawlor <jonathan.lawlor@gmail.com> | Jonathan Reiter <jonreiter@gmail.com> | Jonathan Schroeder <jd.schroeder@gmail.com> | Joost van Amersfoort <git@joo.st> | Jordan Stoker <jordan_stoker@hotmail.com> | Joseph Watson <jtwatson@linux-consulting.us> | Josh Wilson <josh.craig.wilson@gmail.com> | Julien Roland <juroland@gmail.com> | Kai Trukenmüller <ktye78@gmail.com> | Kent English <kent.english@gmail.com> | Kevin C. Zimmerman <kevinczimmerman@gmail.com> | Kirill Motkov <motkov.kirill@gmail.com> | Konstantin Shaposhnikov <k.shaposhnikov@gmail.com> | Leonid Kneller <recondite.matter@gmail.com> | Lyron Winderbaum <lyron.winderbaum@student.adelaide.edu.au> <armadilloa16@gmail.com> <lyron.winderbaum@uwa.edu.au> | Marco Leogrande <dark.knight.ita@gmail.com> | Mark Canning <argusdusty@gmail.com> | Mark Skilbeck <markskilbeck@gmail.com> | Martin Diz <github@martindiz.com.ar> | Matthew Connelly <matthew.b.connelly@gmail.com> | Matthieu Di Mercurio <matthieu.dimercurio@gmail.com> | Max Halford <maxhalford25@gmail.com> | Maxim Sergeev <gudvinr@gmail.com> | Microsoft Corporation | MinJae Kwon <k239507@gmail.com> | Nathan Edwards <etaoinshrdluwho@gmail.com> | Nick Potts <nick@the-potts.com> | Nils Wogatzky <odog@netcologne.de> | Olivier Wulveryck <olivier.wulveryck@gmail.com> | Or Rikon <rikonor@gmail.com> | Patricio Whittingslow <graded.sp@gmail.com> | Patrick DeVivo <patrick@tickgit.com> | Pontus Melke <pontusmelke@gmail.com> | Renee French | Rishi Desai <desai.rishi1@gmail.com> | Robin Eklind <r.eklind.87@gmail.com> | Roger Welin <roger.welin@icloud.com> | Roman Werpachowski <roman.werpachowski@gmail.com> | Rondall Jones <rejones7@gmail.com> | Sam Zaydel <szaydel@gmail.com> | Samuel Kelemen <Samuel@Kelemen.us> | Saran Ahluwalia <ahlusar.ahluwalia@gmail.com> | Scott Holden <scott@sshconnection.com> | Scott Kiesel <kiesel.scott@gmail.com> | Sebastien Binet <seb.binet@gmail.com> | Shawn Smith <shawnpsmith@gmail.com> | Sintela Ltd | Spencer Lyon <spencerlyon2@gmail.com> | Steve McCoy <mccoyst@gmail.com> | Taesu Pyo <pyotaesu@gmail.com> | Takeshi Yoneda <cz.rk.t0415y.g@gmail.com> | Tamir Hyman <hyman.tamir@gmail.com> | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg <tomfuture@gmail.com> | Tobin Harding <me@tobin.cc> | Valentin Deleplace <deleplace2015@gmail.com> | Vincent Thiery <vjmthiery@gmail.com> | Vladimír Chalupecký <vladimir.chalupecky@gmail.com> | Will Tekulve <tekulve.will@gmail.com> | Yasuhiro Matsumoto <mattn.jp@gmail.com> | Yevgeniy Vahlis <evahlis@gmail.com> | Yucheng Zhu <zyctc000@gmail.com> | Yunomi <ynmtywn@gmail.com> | Zoe Juozapaitis | antichris <chris@u-d13.com> | source{d} <hello@sourced.tech>
 core,gonum.org/v1/gonum/blas/blas64,BSD-3-Clause,Alexander Egurnov <alexander.egurnov@gmail.com> | Andrei Blinnikov <goofinator@mail.ru> | Andrew Brampton <brampton@gmail.com> | Bailey Lissington <lissington4@gmail.com> | Bill Gray <wgray@gogray.com> | Bill Noon <noon.bill@gmail.com> | Brendan Tracey <tracey.brendan@gmail.com> | Brent Pedersen <bpederse@gmail.com> | Bulat Khasanov <afti@yandex.ru> | Chad Kunde <kunde21@gmail.com> | Chan Kwan Yin <sofe2038@gmail.com> | Chih-Wei Chang <bert.cwchang@gmail.com> | Chong-Yeol Nah <nahchongyeol@gmail.com> | Chris Tessum <ctessum@gmail.com> | Christophe Meessen <christophe.meessen@gmail.com> | Christopher Waldon <christopher.waldon.dev@gmail.com> | Clayton Northey <clayton.northey@gmail.com> | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak <dan.kortschak@adelaide.edu.au> <dan@kortschak.io> | Dan Lorenc <lorenc.d@gmail.com> | Daniel Fireman <danielfireman@gmail.com> | Dario Heinisch <dario.heinisch@gmail.com> | David Kleiven <davidkleiven446@gmail.com> | David Samborski <bloggingarrow@gmail.com> | Davor Kapsa <davor.kapsa@gmail.com> | DeepMind Technologies | Delaney Gillilan <delaneygillilan@gmail.com> | Dezmond Goff <goff.dezmond@gmail.com> | Dong-hee Na <donghee.na92@gmail.com> | Dustin Spicuzza <dustin@virtualroadside.com> | Egon Elbre <egonelbre@gmail.com> | Ekaterina Efimova <katerina.efimova@gmail.com> | Eng Zer Jun <engzerjun@gmail.com> | Ethan Burns <burns.ethan@gmail.com> | Ethan Reesor <ethan.reesor@gmail.com> | Evert Lammerts <evert.lammerts@gmail.com> | Evgeny Savinov <notime.sea@gmail.com> | Fabian Wickborn <fabian@wickborn.net> | Facundo Gaich <facugaich@gmail.com> | Fazlul Shahriar <fshahriar@gmail.com> | Francesc Campoy <campoy@golang.org> | Google Inc | Gustaf Johansson <gustaf@pinon.se> | Hossein Zolfi <hossein.zolfi@gmail.com> | Iakov Davydov <iakov.davydov@unil.ch> | Igor Mikushkin <igor.mikushkin@gmail.com> | Iskander Sharipov <quasilyte@gmail.com> | Jalem Raj Rohit <jrajrohit33@gmail.com> | James Bell <james@stellentus.com> | James Bowman <james.edward.bowman@gmail.com> | James Holmes <32bitkid@gmail.com> | Janne Snabb <snabb@epipe.com> | Jeremy Atkinson <jchatkinson@gmail.com> | Jes Cok <xigua67damn@gmail.com> | Jinesi Yelizati <i63888888@163.com> | Jon Richards <noj.richards@gmail.com> | Jonas Kahler <jonas@derkahler.de> | Jonas Schulze <jonas.schulze@ovgu.de> | Jonathan Bluett-Duncan <jbluettduncan@gmail.com> | Jonathan J Lawlor <jonathan.lawlor@gmail.com> | Jonathan Reiter <jonreiter@gmail.com> | Jonathan Schroeder <jd.schroeder@gmail.com> | Joost van Amersfoort <git@joo.st> | Jordan Stoker <jordan_stoker@hotmail.com> | Joseph Watson <jtwatson@linux-consulting.us> | Josh Wilson <josh.craig.wilson@gmail.com> | Julien Roland <juroland@gmail.com> | Kai Trukenmüller <ktye78@gmail.com> | Kent English <kent.english@gmail.com> | Kevin C. Zimmerman <kevinczimmerman@gmail.com> | Kirill Motkov <motkov.kirill@gmail.com> | Konstantin Shaposhnikov <k.shaposhnikov@gmail.com> | Leonid Kneller <recondite.matter@gmail.com> | Lyron Winderbaum <lyron.winderbaum@student.adelaide.edu.au> <armadilloa16@gmail.com> <lyron.winderbaum@uwa.edu.au> | Marco Leogrande <dark.knight.ita@gmail.com> | Mark Canning <argusdusty@gmail.com> | Mark Skilbeck <markskilbeck@gmail.com> | Martin Diz <github@martindiz.com.ar> | Matthew Connelly <matthew.b.connelly@gmail.com> | Matthieu Di Mercurio <matthieu.dimercurio@gmail.com> | Max Halford <maxhalford25@gmail.com> | Maxim Sergeev <gudvinr@gmail.com> | Microsoft Corporation | MinJae Kwon <k239507@gmail.com> | Nathan Edwards <etaoinshrdluwho@gmail.com> | Nick Potts <nick@the-potts.com> | Nils Wogatzky <odog@netcologne.de> | Olivier Wulveryck <olivier.wulveryck@gmail.com> | Or Rikon <rikonor@gmail.com> | Patricio Whittingslow <graded.sp@gmail.com> | Patrick DeVivo <patrick@tickgit.com> | Pontus Melke <pontusmelke@gmail.com> | Renee French | Rishi Desai <desai.rishi1@gmail.com> | Robin Eklind <r.eklind.87@gmail.com> | Roger Welin <roger.welin@icloud.com> | Roman Werpachowski <roman.werpachowski@gmail.com> | Rondall Jones <rejones7@gmail.com> | Sam Zaydel <szaydel@gmail.com> | Samuel Kelemen <Samuel@Kelemen.us> | Saran Ahluwalia <ahlusar.ahluwalia@gmail.com> | Scott Holden <scott@sshconnection.com> | Scott Kiesel <kiesel.scott@gmail.com> | Sebastien Binet <seb.binet@gmail.com> | Shawn Smith <shawnpsmith@gmail.com> | Sintela Ltd | Spencer Lyon <spencerlyon2@gmail.com> | Steve McCoy <mccoyst@gmail.com> | Taesu Pyo <pyotaesu@gmail.com> | Takeshi Yoneda <cz.rk.t0415y.g@gmail.com> | Tamir Hyman <hyman.tamir@gmail.com> | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg <tomfuture@gmail.com> | Tobin Harding <me@tobin.cc> | Valentin Deleplace <deleplace2015@gmail.com> | Vincent Thiery <vjmthiery@gmail.com> | Vladimír Chalupecký <vladimir.chalupecky@gmail.com> | Will Tekulve <tekulve.will@gmail.com> | Yasuhiro Matsumoto <mattn.jp@gmail.com> | Yevgeniy Vahlis <evahlis@gmail.com> | Yucheng Zhu <zyctc000@gmail.com> | Yunomi <ynmtywn@gmail.com> | Zoe Juozapaitis | antichris <chris@u-d13.com> | source{d} <hello@sourced.tech>

cmd/agent/command/command.go

@@ -87,6 +87,7 @@ monitoring and performance data.`,
 	agentCmd.PersistentFlags().StringArrayVarP(&globalParams.ExtraConfFilePath, "extracfgpath", "E", []string{}, "specify additional configuration files to be loaded sequentially after the main datadog.yaml")
 	agentCmd.PersistentFlags().StringVarP(&globalParams.SysProbeConfFilePath, "sysprobecfgpath", "", "", "path to directory containing system-probe.yaml")
 	agentCmd.PersistentFlags().StringVarP(&globalParams.FleetPoliciesDirPath, "fleetcfgpath", "", "", "path to the directory containing fleet policies")
+	_ = agentCmd.PersistentFlags().MarkHidden("fleetcfgpath")
 
 	// github.com/fatih/color sets its global color.NoColor to a default value based on
 	// whether the process is running in a tty.  So, we only want to override that when

cmd/process-agent/command/command.go

@@ -88,6 +88,7 @@ func MakeCommand(subcommandFactories []SubcommandFactory, winParams bool, rootCm
 
 	rootCmd.PersistentFlags().StringVar(&globalParams.ConfFilePath, flags.CfgPath, flags.DefaultConfPath, "Path to datadog.yaml config")
 	rootCmd.PersistentFlags().StringVar(&globalParams.FleetPoliciesDirPath, flags.FleetCfgPath, "", "Path to the directory containing fleet policies")
+	_ = rootCmd.PersistentFlags().MarkHidden(flags.FleetCfgPath)
 
 	if flags.DefaultSysProbeConfPath != "" {
 		rootCmd.PersistentFlags().StringVar(&globalParams.SysProbeConfFilePath, flags.SysProbeConfig, flags.DefaultSysProbeConfPath, "Path to system-probe.yaml config")

cmd/security-agent/command/command.go

@@ -70,6 +70,7 @@ Datadog Security Agent takes care of running compliance and security checks.`,
 	SecurityAgentCmd.PersistentFlags().StringVar(&globalParams.SysProbeConfFilePath, "sysprobe-config", defaultSysProbeConfPath, "path to system-probe.yaml config")
 	SecurityAgentCmd.PersistentFlags().BoolVarP(&globalParams.NoColor, "no-color", "n", false, "disable color output")
 	SecurityAgentCmd.PersistentFlags().StringVar(&globalParams.FleetPoliciesDirPath, "fleetcfgpath", "", "path to the directory containing fleet policies")
+	_ = SecurityAgentCmd.PersistentFlags().MarkHidden("fleetcfgpath")
 
 	for _, factory := range subcommandFactories {
 		for _, subcmd := range factory(&globalParams) {

cmd/system-probe/command/command.go

@@ -48,6 +48,7 @@ Runtime Security Monitoring, Universal Service Monitoring, and others.`,
 
 	sysprobeCmd.PersistentFlags().StringVarP(&globalParams.ConfFilePath, "config", "c", "", "path to directory containing system-probe.yaml")
 	sysprobeCmd.PersistentFlags().StringVarP(&globalParams.FleetPoliciesDirPath, "fleetcfgpath", "", "", "path to the directory containing fleet policies")
+	_ = sysprobeCmd.PersistentFlags().MarkHidden("fleetcfgpath")
 
 	// github.com/fatih/color sets its global color.NoColor to a default value based on
 	// whether the process is running in a tty.  So, we only want to override that when

cmd/trace-agent/command/command.go

@@ -62,6 +62,7 @@ func makeCommands(globalParams *subcommands.GlobalParams) *cobra.Command {
 
 	traceAgentCmd.PersistentFlags().StringVarP(&globalParams.ConfPath, "config", "c", defaultConfigPath, "path to directory containing datadog.yaml")
 	traceAgentCmd.PersistentFlags().StringVarP(&globalParams.FleetPoliciesDirPath, "fleetcfgpath", "", "", "path to the directory containing fleet policies")
+	_ = traceAgentCmd.PersistentFlags().MarkHidden("fleetcfgpath")
 
 	return &traceAgentCmd
 }

cmd/trace-agent/test/testsuite/stats_test.go

@@ -6,6 +6,8 @@
 package testsuite
 
 import (
+	"cmp"
+	"slices"
 	"testing"
 	"time"
 
@@ -54,6 +56,25 @@ func TestClientStats(t *testing.T) {
 						continue
 					}
 					assert.Equalf(t, len(res), len(tt.Out), "res had so many elements: %d\ntt has:%d", len(res), len(tt.Out))
+
+					// sort internal slices for testing ease
+					for _, r := range res {
+						for _, p := range r.Stats {
+							sortBuckets(p.Stats)
+							for _, b := range p.Stats {
+								sortStats(b.Stats)
+							}
+						}
+					}
+					for _, r := range tt.Out {
+						for _, p := range r.Stats {
+							sortBuckets(p.Stats)
+							for _, b := range p.Stats {
+								sortStats(b.Stats)
+							}
+						}
+					}
+
 					actual := []protoiface.MessageV1{}
 					expected := []protoiface.MessageV1{}
 					for _, msg := range res {
@@ -72,11 +93,28 @@ func TestClientStats(t *testing.T) {
 	}
 }
 
+func sortBuckets(buckets []*pb.ClientStatsBucket) {
+	slices.SortFunc(buckets, func(a, b *pb.ClientStatsBucket) int {
+		return cmp.Compare(a.Start, b.Start)
+	})
+}
+
+func sortStats(stats []*pb.ClientGroupedStats) {
+	slices.SortFunc(stats, func(a, b *pb.ClientGroupedStats) int {
+		if n := cmp.Compare(a.Service, b.Service); n != 0 {
+			return n
+		}
+		if n := cmp.Compare(a.Name, b.Name); n != 0 {
+			return n
+		}
+		return cmp.Compare(a.Resource, b.Resource)
+	})
+}
+
 func normalizeTimeFields(t *testing.T, p *pb.StatsPayload) *pb.StatsPayload {
 	now := time.Now().UnixNano()
 	for _, s := range p.Stats {
 		for i := range s.Stats {
-			assert.True(t, s.Stats[i].AgentTimeShift > now-100*1e9)
 			s.Stats[i].AgentTimeShift = 0
 			assert.True(t, s.Stats[i].Start >= uint64(now-40*1e9))
 			s.Stats[i].Start = 0