V1 metrics monitoring for V2

go.mod

@@ -14,12 +14,14 @@ require (
 	github.com/elastic/elastic-agent-autodiscover v0.0.0-20220404145827-89887023c1ab
 	github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484
 	github.com/elastic/elastic-agent-libs v0.2.6
+	github.com/elastic/elastic-agent-system-metrics v0.4.4
 	github.com/elastic/go-licenser v0.4.0
 	github.com/elastic/go-sysinfo v1.7.1
 	github.com/elastic/go-ucfg v0.8.5
 	github.com/gofrs/flock v0.8.1
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/google/go-cmp v0.5.6
+	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901
@@ -38,6 +40,7 @@ require (
 	github.com/spf13/cobra v1.3.0
 	github.com/stretchr/testify v1.7.0
 	github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
+	go.elastic.co/apm/module/apmgorilla v1.15.0
 	go.elastic.co/ecszap v1.0.1
 	go.elastic.co/go-licence-detector v0.5.0
 	go.uber.org/zap v1.21.0
@@ -69,7 +72,9 @@ require (
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/docker v20.10.12+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/elastic/go-structform v0.0.9 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect

go.sum

@@ -383,12 +383,16 @@ github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go
 github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM=
 github.com/elastic/elastic-agent-libs v0.2.6 h1:DpcUcCVYZ7lNtHLUlyT1u/GtGAh49wpL15DTH7+8O5o=
 github.com/elastic/elastic-agent-libs v0.2.6/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
+github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0=
+github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
 github.com/elastic/elastic-package v0.32.1/go.mod h1:l1fEnF52XRBL6a5h6uAemtdViz2bjtjUtgdQcuRhEAY=
 github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-elasticsearch/v8 v8.0.0-20210317102009-a9d74cec0186/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4=
 github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
 github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI=
 github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU=
+github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o=
+github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
 github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
 github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0=
 github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
@@ -398,6 +402,8 @@ github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
+github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
+github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/package-spec v1.3.0/go.mod h1:KzGTSDqCkdhmL1IFpOH2ZQNSSE9JEhNtndxU3ZrQilA=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -634,8 +640,10 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c
 github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
+github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
@@ -1189,6 +1197,8 @@ go.elastic.co/apm v1.13.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY=
 go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q=
 go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY=
 go.elastic.co/apm/module/apmelasticsearch v1.10.0/go.mod h1:lwoaGDfZzfb9e6TXd3h8/KNmLAONOas7o5NLVNmv8Xk=
+go.elastic.co/apm/module/apmgorilla v1.15.0 h1:1yTAksffgaFXYEIwlLRiQnxLfy3p3RtpDw8HDupIJfY=
+go.elastic.co/apm/module/apmgorilla v1.15.0/go.mod h1:+23mZudYvZ9VgxCQjseLo9EF5gkKEr0KSQBupw+rzP8=
 go.elastic.co/apm/module/apmgrpc v1.15.0 h1:Z7h58uuMJUoYXK6INFunlcGEXZQ18QKAhPh6NFYDNHE=
 go.elastic.co/apm/module/apmgrpc v1.15.0/go.mod h1:IEbTGJzY5Xx737PkHDT3bbzh9syovK+IfAlckJsUgPE=
 go.elastic.co/apm/module/apmhttp v1.10.0/go.mod h1:Y4timwcJ8sQWbWpcw3Y7Mat1OssNpGhpwyfUnpqIDew=
@@ -1429,6 +1439,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

internal/pkg/agent/application/application.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/monitoring"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
@@ -64,8 +65,9 @@ func New(
 	}
 
 	upgrader := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, agentInfo)
+	monitor := monitoring.New(cfg.Settings.V1MonitoringEnabled, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, agentInfo)
 
-	runtime, err := runtime.NewManager(log, cfg.Settings.GRPC.String(), agentInfo, tracer)
+	runtime, err := runtime.NewManager(log, cfg.Settings.GRPC.String(), agentInfo, tracer, monitor)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize runtime manager: %w", err)
 	}
@@ -114,7 +116,7 @@ func New(
 		return nil, errors.New(err, "failed to initialize composable controller")
 	}
 
-	coord := coordinator.New(log, agentInfo, specs, reexec, upgrader, runtime, configMgr, composable, caps, compModifiers...)
+	coord := coordinator.New(log, agentInfo, specs, reexec, upgrader, runtime, configMgr, composable, caps, monitor, compModifiers...)
 	if managed != nil {
 		// the coordinator requires the config manager as well as in managed-mode the config manager requires the
 		// coordinator, so it must be set here once the coordinator is created

internal/pkg/agent/application/coordinator/coordinator.go

@@ -53,6 +53,18 @@ type UpgradeManager interface {
 	Upgrade(ctx context.Context, version string, sourceURI string, action *fleetapi.ActionUpgrade) (_ reexec.ShutdownCallbackFn, err error)
 }
 
+// MonitorManager provides an interface to perform the monitoring action for the agent.
+type MonitorManager interface {
+	// Enabled when configured to collect metrics/logs.
+	Enabled() bool
+
+	// Reload reloads the configuration for the upgrade manager.
+	Reload(rawConfig *config.Config) error
+
+	// InjectMonitoring injects monitoring configuration into resolved ast tree.
+	MonitoringConfig(map[string]interface{}, map[string]string) (map[string]interface{}, error)
+}
+
 // Runner provides interface to run a manager and receive running errors.
 type Runner interface {
 	// Run runs the manager.
@@ -147,6 +159,7 @@ type Coordinator struct {
 
 	reexecMgr  ReExecManager
 	upgradeMgr UpgradeManager
+	monitorMgr MonitorManager
 
 	runtimeMgr    RuntimeManager
 	runtimeMgrErr error
@@ -162,7 +175,7 @@ type Coordinator struct {
 }
 
 // New creates a new coordinator.
-func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.RuntimeSpecs, reexecMgr ReExecManager, upgradeMgr UpgradeManager, runtimeMgr RuntimeManager, configMgr ConfigManager, varsMgr VarsManager, caps capabilities.Capability, modifiers ...ComponentsModifier) *Coordinator {
+func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.RuntimeSpecs, reexecMgr ReExecManager, upgradeMgr UpgradeManager, runtimeMgr RuntimeManager, configMgr ConfigManager, varsMgr VarsManager, caps capabilities.Capability, monitorMgr MonitorManager, modifiers ...ComponentsModifier) *Coordinator {
 	return &Coordinator{
 		logger:     logger,
 		agentInfo:  agentInfo,
@@ -177,6 +190,7 @@ func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.Runti
 		state: coordinatorState{
 			state: agentclient.Starting,
 		},
+		monitorMgr: monitorMgr,
 	}
 }
 
@@ -565,6 +579,10 @@ func (c *Coordinator) processConfig(ctx context.Context, cfg *config.Config) (er
 		return fmt.Errorf("failed to reload upgrade manager configuration: %w", err)
 	}
 
+	if err := c.monitorMgr.Reload(cfg); err != nil {
+		return fmt.Errorf("failed to reload upgrade manager configuration: %w", err)
+	}
+
 	c.state.config = cfg
 	c.state.ast = rawAst
 
@@ -630,7 +648,13 @@ func (c *Coordinator) compute() (map[string]interface{}, []component.Component,
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to convert ast to map[string]interface{}: %w", err)
 	}
-	comps, err := c.specs.ToComponents(cfg)
+
+	var configInjector component.GenerateMonitoringCfgFn
+	if c.monitorMgr.Enabled() {
+		configInjector = c.monitorMgr.MonitoringConfig
+	}
+
+	comps, err := c.specs.ToComponents(cfg, configInjector)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to render components: %w", err)
 	}

internal/pkg/agent/application/monitoring/handler.go

@@ -0,0 +1,70 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package monitoring
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+const errTypeUnexpected = "UNEXPECTED"
+
+type apiError interface {
+	Status() int
+}
+
+func createHandler(fn func(w http.ResponseWriter, r *http.Request) error) *apiHandler {
+	return &apiHandler{
+		innerFn: fn,
+	}
+}
+
+type apiHandler struct {
+	innerFn func(w http.ResponseWriter, r *http.Request) error
+}
+
+// ServeHTTP sets status code based on err returned
+func (h *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	err := h.innerFn(w, r)
+	if err != nil {
+		switch e := err.(type) { // nolint:errorlint // Will need refactor.
+		case apiError:
+			w.WriteHeader(e.Status())
+		default:
+			w.WriteHeader(http.StatusInternalServerError)
+
+		}
+
+		writeResponse(w, unexpectedErrorWithReason(err.Error()))
+	}
+}
+
+func writeResponse(w http.ResponseWriter, c interface{}) {
+	bytes, err := json.Marshal(c)
+	if err != nil {
+		// json marshal failed
+		fmt.Fprintf(w, "Not valid json: %v", err)
+		return
+	}
+
+	fmt.Fprint(w, string(bytes))
+
+}
+
+type errResponse struct {
+	// Type is a type of error
+	Type string `json:"type"`
+
+	// Reason is a detailed error message
+	Reason string `json:"reason"`
+}
+
+func unexpectedErrorWithReason(reason string, args ...interface{}) errResponse {
+	return errResponse{
+		Type:   errTypeUnexpected,
+		Reason: fmt.Sprintf(reason, args...),
+	}
+}

internal/pkg/agent/application/monitoring/server.go

@@ -0,0 +1,86 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package monitoring
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"go.elastic.co/apm"
+	"go.elastic.co/apm/module/apmgorilla"
+
+	"github.com/elastic/elastic-agent-libs/api"
+	"github.com/elastic/elastic-agent-libs/config"
+	"github.com/elastic/elastic-agent-libs/monitoring"
+	"github.com/elastic/elastic-agent/pkg/core/logger"
+)
+
+// New creates a new server exposing metrics and process information.
+func NewServer(
+	log *logger.Logger,
+	endpointConfig api.Config,
+	ns func(string) *monitoring.Namespace,
+	tracer *apm.Tracer,
+) (*api.Server, error) {
+	if err := createAgentMonitoringDrop(endpointConfig.Host); err != nil {
+		// log but ignore
+		log.Errorf("failed to create monitoring drop: %v", err)
+	}
+
+	cfg, err := config.NewConfigFrom(endpointConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return exposeMetricsEndpoint(log, cfg, ns, tracer)
+}
+
+func exposeMetricsEndpoint(
+	log *logger.Logger,
+	config *config.C,
+	ns func(string) *monitoring.Namespace,
+	tracer *apm.Tracer,
+) (*api.Server, error) {
+	r := mux.NewRouter()
+	if tracer != nil {
+		r.Use(apmgorilla.Middleware(apmgorilla.WithTracer(tracer)))
+	}
+	statsHandler := statsHandler(ns("stats"))
+	r.Handle("/stats", createHandler(statsHandler))
+
+	mux := http.NewServeMux()
+	mux.Handle("/", r)
+
+	return api.New(log, mux, config)
+}
+
+func createAgentMonitoringDrop(drop string) error {
+	if drop == "" || runtime.GOOS == "windows" {
+		return nil
+	}
+
+	path := strings.TrimPrefix(drop, "unix://")
+	if strings.HasSuffix(path, ".sock") {
+		path = filepath.Dir(path)
+	}
+
+	_, err := os.Stat(path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+
+		// create
+		if err := os.MkdirAll(path, 0775); err != nil {
+			return err
+		}
+	}
+
+	return os.Chown(path, os.Geteuid(), os.Getegid())
+}

internal/pkg/agent/application/monitoring/stats.go

@@ -0,0 +1,36 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package monitoring
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/elastic/elastic-agent-libs/monitoring"
+)
+
+func statsHandler(ns *monitoring.Namespace) func(http.ResponseWriter, *http.Request) error {
+	return func(w http.ResponseWriter, r *http.Request) error {
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+		data := monitoring.CollectStructSnapshot(
+			ns.GetRegistry(),
+			monitoring.Full,
+			false,
+		)
+
+		bytes, err := json.Marshal(data)
+		var content string
+		if err != nil {
+			content = fmt.Sprintf("Not valid json: %v", err)
+		} else {
+			content = string(bytes)
+		}
+		fmt.Fprint(w, content)
+
+		return nil
+	}
+}