upgrade(fleet): Plug rudimentary RC client for policies (#28482)

Co-authored-by: arbll <arthur.bellal@datadoghq.com>

comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl/rctelemetryreporter.go

@@ -35,12 +35,16 @@ type DdRcTelemetryReporter struct {
 
 // IncRateLimit increments the DdRcTelemetryReporter BypassRateLimitCounter counter.
 func (r *DdRcTelemetryReporter) IncRateLimit() {
-	r.BypassRateLimitCounter.Inc()
+	if r.BypassRateLimitCounter != nil {
+		r.BypassRateLimitCounter.Inc()
+	}
 }
 
 // IncTimeout increments the DdRcTelemetryReporter BypassTimeoutCounter counter.
 func (r *DdRcTelemetryReporter) IncTimeout() {
-	r.BypassTimeoutCounter.Inc()
+	if r.BypassTimeoutCounter != nil {
+		r.BypassTimeoutCounter.Inc()
+	}
 }
 
 // newDdRcTelemetryReporter creates a new Remote Config telemetry reporter for sending RC metrics to Datadog

pkg/fleet/installer/service/datadog_agent.go

@@ -24,20 +24,22 @@ import (
 )
 
 const (
-	agentPackage      = "datadog-agent"
-	pathOldAgent      = "/opt/datadog-agent"
-	agentSymlink      = "/usr/bin/datadog-agent"
-	agentUnit         = "datadog-agent.service"
-	traceAgentUnit    = "datadog-agent-trace.service"
-	processAgentUnit  = "datadog-agent-process.service"
-	systemProbeUnit   = "datadog-agent-sysprobe.service"
-	securityAgentUnit = "datadog-agent-security.service"
-	agentExp          = "datadog-agent-exp.service"
-	traceAgentExp     = "datadog-agent-trace-exp.service"
-	processAgentExp   = "datadog-agent-process-exp.service"
-	systemProbeExp    = "datadog-agent-sysprobe-exp.service"
-	securityAgentExp  = "datadog-agent-security-exp.service"
-	configDatadogYAML = "datadog.yaml"
+	agentPackage            = "datadog-agent"
+	pathOldAgent            = "/opt/datadog-agent"
+	agentSymlink            = "/usr/bin/datadog-agent"
+	agentUnit               = "datadog-agent.service"
+	traceAgentUnit          = "datadog-agent-trace.service"
+	processAgentUnit        = "datadog-agent-process.service"
+	systemProbeUnit         = "datadog-agent-sysprobe.service"
+	securityAgentUnit       = "datadog-agent-security.service"
+	agentExp                = "datadog-agent-exp.service"
+	traceAgentExp           = "datadog-agent-trace-exp.service"
+	processAgentExp         = "datadog-agent-process-exp.service"
+	systemProbeExp          = "datadog-agent-sysprobe-exp.service"
+	securityAgentExp        = "datadog-agent-security-exp.service"
+	configDatadogYAML       = "datadog.yaml"
+	configSecurityAgentYAML = "security-agent.yaml"
+	configSystemProbeYAML   = "system-probe.yaml"
 )
 
 var (
@@ -242,14 +244,30 @@ func ConfigureAgent(ctx context.Context, cdn *cdn.CDN, configs *repository.Repos
 	if err != nil {
 		return fmt.Errorf("error getting dd-agent user and group IDs: %w", err)
 	}
-	err = os.WriteFile(filepath.Join(tmpDir, configDatadogYAML), []byte(config.Datadog), 0640)
-	if err != nil {
-		return fmt.Errorf("could not write datadog.yaml: %w", err)
+
+	if config.Datadog != nil {
+		err = os.WriteFile(filepath.Join(tmpDir, configDatadogYAML), []byte(config.Datadog), 0640)
+		if err != nil {
+			return fmt.Errorf("could not write datadog.yaml: %w", err)
+		}
+		err = os.Chown(filepath.Join(tmpDir, configDatadogYAML), ddAgentUID, ddAgentGID)
+		if err != nil {
+			return fmt.Errorf("could not chown datadog.yaml: %w", err)
+		}
 	}
-	err = os.Chown(filepath.Join(tmpDir, configDatadogYAML), ddAgentUID, ddAgentGID)
-	if err != nil {
-		return fmt.Errorf("could not chown datadog.yaml: %w", err)
+	if config.SecurityAgent != nil {
+		err = os.WriteFile(filepath.Join(tmpDir, configSecurityAgentYAML), []byte(config.SecurityAgent), 0600)
+		if err != nil {
+			return fmt.Errorf("could not write datadog.yaml: %w", err)
+		}
+	}
+	if config.SystemProbe != nil {
+		err = os.WriteFile(filepath.Join(tmpDir, configSystemProbeYAML), []byte(config.SystemProbe), 0600)
+		if err != nil {
+			return fmt.Errorf("could not write datadog.yaml: %w", err)
+		}
 	}
+
 	err = configs.Create(agentPackage, config.Version, tmpDir)
 	if err != nil {
 		return fmt.Errorf("could not create repository: %w", err)

pkg/fleet/internal/cdn/cdn.go

@@ -8,13 +8,30 @@ package cdn
 
 import (
 	"context"
-	"crypto/sha256"
+	"encoding/json"
 	"fmt"
+	"regexp"
+	"time"
 
+	"github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags"
+	"github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl"
+	detectenv "github.com/DataDog/datadog-agent/pkg/config/env"
+	"github.com/DataDog/datadog-agent/pkg/config/model"
+	remoteconfig "github.com/DataDog/datadog-agent/pkg/config/remote/service"
+	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
 	"github.com/DataDog/datadog-agent/pkg/fleet/env"
+	pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core"
+	pkghostname "github.com/DataDog/datadog-agent/pkg/util/hostname"
+	"github.com/DataDog/datadog-agent/pkg/version"
+	"github.com/google/uuid"
+	"go.uber.org/multierr"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 )
 
+var datadogConfigIDRegexp = regexp.MustCompile(`^datadog/\d+/AGENT_CONFIG/([^/]+)/[^/]+$`)
+
+const configOrderID = "configuration_order"
+
 // CDN provides access to the Remote Config CDN.
 type CDN struct {
 	env *env.Env
@@ -28,6 +45,10 @@ type Config struct {
 	SystemProbe   []byte
 }
 
+type orderConfig struct {
+	Order []string `json:"order"`
+}
+
 // New creates a new CDN.
 func New(env *env.Env) *CDN {
 	return &CDN{
@@ -36,38 +57,124 @@ func New(env *env.Env) *CDN {
 }
 
 // Get gets the configuration from the CDN.
-func (c *CDN) Get(ctx context.Context) (_ Config, err error) {
+func (c *CDN) Get(ctx context.Context) (_ *Config, err error) {
 	span, ctx := tracer.StartSpanFromContext(ctx, "cdn.Get")
 	defer func() { span.Finish(tracer.WithError(err)) }()
-	return getFakeCDNConfig(ctx)
+	configLayers, err := c.getOrderedLayers(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return newConfig(configLayers...)
 }
 
-// HACK (arthur): this is a temporary function that returns a fake CDN config to unblock development.
-func getFakeCDNConfig(_ context.Context) (Config, error) {
-	baseLayer := layer{
-		ID: "base",
-		Content: map[interface{}]interface{}{
-			"extra_tags": []string{"layer:base"},
-		},
+// getOrderedLayers calls the Remote Config service to get the ordered layers.
+// Today it doesn't use the CDN, but it should in the future
+func (c *CDN) getOrderedLayers(ctx context.Context) ([]*layer, error) {
+	// HACK(baptiste): Create a dedicated one-shot RC service just for the configuration
+	// We should use the CDN instead
+	config := pkgconfigsetup.Datadog()
+	config.Set("run_path", "/opt/datadog-packages/datadog-installer/stable/run", model.SourceAgentRuntime)
+
+	detectenv.DetectFeatures(config)
+	hostname, err := pkghostname.Get(ctx)
+	if err != nil {
+		return nil, err
 	}
-	overrideLayer := layer{
-		ID: "override",
-		Content: map[interface{}]interface{}{
-			"extra_tags": []string{"layer:override"},
-		},
+	options := []remoteconfig.Option{
+		remoteconfig.WithAPIKey(c.env.APIKey),
+		remoteconfig.WithConfigRootOverride(c.env.Site, ""),
+		remoteconfig.WithDirectorRootOverride(c.env.Site, ""),
 	}
-	config, err := newConfig(baseLayer, overrideLayer)
+	service, err := remoteconfig.NewService(
+		config,
+		"Datadog Installer",
+		fmt.Sprintf("https://config.%s", c.env.Site),
+		hostname,
+		getHostTags(ctx, config),
+		&rctelemetryreporterimpl.DdRcTelemetryReporter{}, // No telemetry for this client
+		version.AgentVersion,
+		options...,
+	)
 	if err != nil {
-		return Config{}, err
+		return nil, err
 	}
-	serializedConfig, err := config.Marshal()
+	service.Start()
+	defer func() { _ = service.Stop() }()
+	// Force a cache bypass
+	cfgs, err := service.ClientGetConfigs(ctx, &pbgo.ClientGetConfigsRequest{
+		Client: &pbgo.Client{
+			Id:            uuid.New().String(),
+			Products:      []string{"AGENT_CONFIG"},
+			IsUpdater:     true,
+			ClientUpdater: &pbgo.ClientUpdater{},
+			State: &pbgo.ClientState{
+				RootVersion:    1,
+				TargetsVersion: 1,
+			},
+		},
+	})
 	if err != nil {
-		return Config{}, err
+		return nil, err
+	}
+
+	// Unmarshal RC results
+	configLayers := map[string]*layer{}
+	var configOrder *orderConfig
+	var layersErr error
+	for _, file := range cfgs.TargetFiles {
+		matched := datadogConfigIDRegexp.FindStringSubmatch(file.GetPath())
+		if len(matched) != 2 {
+			layersErr = multierr.Append(layersErr, fmt.Errorf("invalid config path: %s", file.GetPath()))
+			continue
+		}
+		configName := matched[1]
+
+		if configName != configOrderID {
+			configLayer := &layer{}
+			err = json.Unmarshal(file.GetRaw(), configLayer)
+			if err != nil {
+				// If a layer is wrong, fail later to parse the rest and check them all
+				layersErr = multierr.Append(layersErr, err)
+				continue
+			}
+			configLayers[configName] = configLayer
+		} else {
+			configOrder = &orderConfig{}
+			err = json.Unmarshal(file.GetRaw(), configOrder)
+			if err != nil {
+				// Return first - we can't continue without the order
+				return nil, err
+			}
+		}
+	}
+	if layersErr != nil {
+		return nil, layersErr
+	}
+
+	// Order configs
+	if configOrder == nil {
+		return nil, fmt.Errorf("no configuration_order found")
+	}
+	orderedLayers := []*layer{}
+	for _, configName := range configOrder.Order {
+		if configLayer, ok := configLayers[configName]; ok {
+			orderedLayers = append(orderedLayers, configLayer)
+		}
+	}
+
+	return orderedLayers, nil
+}
+
+func getHostTags(ctx context.Context, config model.Config) func() []string {
+	return func() []string {
+		// Host tags are cached on host, but we add a timeout to avoid blocking the RC request
+		// if the host tags are not available yet and need to be fetched. They will be fetched
+		// by the first agent metadata V5 payload.
+		ctx, cc := context.WithTimeout(ctx, time.Second)
+		defer cc()
+		hostTags := hosttags.Get(ctx, true, config)
+		tags := append(hostTags.System, hostTags.GoogleCloudPlatform...)
+		tags = append(tags, "installer:true")
+		return tags
 	}
-	hash := sha256.New()
-	hash.Write(serializedConfig)
-	return Config{
-		Version: fmt.Sprintf("%x", hash.Sum(nil)),
-		Datadog: serializedConfig,
-	}, nil
 }

pkg/fleet/internal/cdn/config.go

@@ -7,6 +7,9 @@ package cdn
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
 
 	"gopkg.in/yaml.v2"
 )
@@ -16,33 +19,85 @@ const (
 	doNotEditDisclaimer = `# This configuration was generated by Datadog's Fleet Automation. DO NOT EDIT.`
 )
 
-// config is a configuration for the package.
-type config map[interface{}]interface{}
-
 // layer is a config layer that can be merged with other layers into a config.
 type layer struct {
-	ID      string
-	Content map[interface{}]interface{}
+	ID                  string                 `json:"name"`
+	AgentConfig         map[string]interface{} `json:"config"`
+	SecurityAgentConfig map[string]interface{} `json:"security_agent"`
+	SystemProbeConfig   map[string]interface{} `json:"system_probe"`
 }
 
 // newConfig creates a new config from a list of layers.
-func newConfig(layers ...layer) (config, error) {
-	config := make(map[interface{}]interface{})
-	var layerIDs []string
+func newConfig(layers ...*layer) (_ *Config, err error) {
+	layerIDs := []string{}
+	mergedLayer := &layer{
+		AgentConfig:         map[string]interface{}{},
+		SecurityAgentConfig: map[string]interface{}{},
+		SystemProbeConfig:   map[string]interface{}{},
+	}
+
+	// Merge all layers in order
 	for _, l := range layers {
-		merged, err := merge(config, l.Content)
-		if err != nil {
-			return nil, err
-		}
-		config = merged.(map[interface{}]interface{})
 		layerIDs = append(layerIDs, l.ID)
+		if l.AgentConfig != nil {
+			agentConfig, err := merge(mergedLayer.AgentConfig, l.AgentConfig)
+			if err != nil {
+				return nil, err
+			}
+			mergedLayer.AgentConfig = agentConfig.(map[string]interface{})
+		}
+
+		if l.SecurityAgentConfig != nil {
+			securityAgentConfig, err := merge(mergedLayer.SecurityAgentConfig, l.SecurityAgentConfig)
+			if err != nil {
+				return nil, err
+			}
+			mergedLayer.SecurityAgentConfig = securityAgentConfig.(map[string]interface{})
+		}
+
+		if l.SystemProbeConfig != nil {
+			systemProbeAgentConfig, err := merge(mergedLayer.SystemProbeConfig, l.SystemProbeConfig)
+			if err != nil {
+				return nil, err
+			}
+			mergedLayer.SystemProbeConfig = systemProbeAgentConfig.(map[string]interface{})
+		}
+	}
+	mergedLayer.AgentConfig[layerKeys] = layerIDs // Add a field with the applied layers that will be reported through inventories
+
+	serializedAgentConfig, err := marshalConfig(mergedLayer.AgentConfig)
+	if err != nil {
+		return nil, err
 	}
-	config[layerKeys] = layerIDs
-	return config, nil
+	serializedSecurityAgentConfig, err := marshalConfig(mergedLayer.SecurityAgentConfig)
+	if err != nil {
+		return nil, err
+	}
+	serializedSystemProbeConfig, err := marshalConfig(mergedLayer.SystemProbeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	hash := sha256.New()
+	serializedConfig, err := json.Marshal(mergedLayer)
+	if err != nil {
+		return nil, err
+	}
+	hash.Write(serializedConfig)
+
+	return &Config{
+		Version:       fmt.Sprintf("%x", hash.Sum(nil)),
+		Datadog:       serializedAgentConfig,
+		SecurityAgent: serializedSecurityAgentConfig,
+		SystemProbe:   serializedSystemProbeConfig,
+	}, nil
 }
 
-// Marshal marshals the config as YAML.
-func (c *config) Marshal() ([]byte, error) {
+// marshalConfig marshals the config as YAML.
+func marshalConfig(c map[string]interface{}) ([]byte, error) {
+	if len(c) == 0 {
+		return nil, nil
+	}
 	var b bytes.Buffer
 	b.WriteString(doNotEditDisclaimer)
 	b.WriteString("\n")